├── docs ├── authors.rst ├── readme.rst ├── changelog.rst ├── contributing.rst ├── requirements.txt ├── reference │ ├── queryjob.rst │ ├── webcaller.rst │ ├── humioexceptions.rst │ ├── humioclient.rst │ └── index.rst ├── spelling_wordlist.txt ├── index.rst └── conf.py ├── src └── humiolib │ ├── __init__.py │ ├── __main__.py │ ├── HumioExceptions.py │ ├── cli.py │ ├── WebCaller.py │ ├── QueryJob.py │ └── HumioClient.py ├── .bumpversion.cfg ├── .editorconfig ├── MANIFEST.in ├── .github ├── pull_request_template.md ├── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md └── workflows │ └── pythonapp.yml ├── AUTHORS.rst ├── setup.cfg ├── .gitignore ├── tests ├── cassettes │ ├── humioclient │ │ ├── test_get_users │ │ ├── test_create_queryjob_incorrect_query_syntax │ │ ├── test_get_status │ │ ├── test_create_queryjob_success │ │ ├── test_ingest_json_on_humioclient_success │ │ ├── test_ingest_json_on_ingestclient_success │ │ ├── test_ingest_messages_on_ingestclient_ssuccess │ │ ├── test_ingest_messages_on_humioclient_success │ │ └── test_streaming_query_success │ └── queryjob │ │ ├── test_poll_until_done_static_queryjob_aggregate_query │ │ ├── test_poll_until_done_static_queryjob_after_fully_polled_fail │ │ ├── test_poll_until_done_live_queryjob │ │ ├── test_poll_until_done_live_queryjob_aggregate_query │ │ ├── test_poll_until_done_static_queryjob_non_aggregate_query │ │ ├── test_poll_until_done_static_queryjob │ │ ├── test_poll_until_done_live_queryjob_non_aggregate_query │ │ ├── test_poll_until_done_live_queryjob_after_fully_polled_fail │ │ └── test_poll_until_done_live_queryjob_poll_after_done_success ├── test_queryjob.py └── test_humioclient.py ├── CHANGELOG.rst ├── setup.py ├── CODE_OF_CONDUCT.md ├── README.rst ├── CONTRIBUTING.rst └── LICENSE /docs/authors.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../AUTHORS.rst 2 | -------------------------------------------------------------------------------- /docs/readme.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CHANGELOG.rst 2 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CONTRIBUTING.rst 2 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx>=1.3 2 | sphinx_rtd_theme 3 | -------------------------------------------------------------------------------- /src/humiolib/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.2.6" 2 | from humiolib.HumioClient import HumioClient, HumioIngestClient -------------------------------------------------------------------------------- /docs/reference/queryjob.rst: -------------------------------------------------------------------------------- 1 | ======== 2 | QueryJob 3 | ======== 4 | .. automodule:: humiolib.QueryJob 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/reference/webcaller.rst: -------------------------------------------------------------------------------- 1 | ========= 2 | WebCaller 3 | ========= 4 | 5 | .. automodule:: humiolib.WebCaller 6 | :members: -------------------------------------------------------------------------------- /docs/reference/humioexceptions.rst: -------------------------------------------------------------------------------- 1 | =============== 2 | HumioExceptions 3 | =============== 4 | 5 | .. automodule:: humiolib.HumioExceptions 6 | :members: -------------------------------------------------------------------------------- /docs/reference/humioclient.rst: -------------------------------------------------------------------------------- 1 | =========== 2 | HumioClient 3 | =========== 4 | .. automodule:: humiolib.HumioClient 5 | :members: 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /docs/reference/index.rst: -------------------------------------------------------------------------------- 1 | Reference 2 | ========= 3 | 4 | .. toctree:: 5 | :glob: 6 | 7 | humioclient* 8 | queryjob* 9 | webcaller* 10 | humioexceptions* 11 | -------------------------------------------------------------------------------- /docs/spelling_wordlist.txt: -------------------------------------------------------------------------------- 1 | builtin 2 | builtins 3 | classmethod 4 | staticmethod 5 | classmethods 6 | staticmethods 7 | args 8 | kwargs 9 | callstack 10 | Changelog 11 | Indices 12 | -------------------------------------------------------------------------------- /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.2.6 3 | tag = True 4 | 5 | [bumpversion:file:setup.py] 6 | 7 | [bumpversion:file:docs/conf.py] 8 | 9 | [bumpversion:file:src/humiolib/__init__.py] 10 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # see https://editorconfig.org/ 2 | root = true 3 | 4 | [*] 5 | end_of_line = lf 6 | trim_trailing_whitespace = true 7 | insert_final_newline = true 8 | indent_style = space 9 | indent_size = 4 10 | charset = utf-8 11 | 12 | [*.{bat,cmd,ps1}] 13 | end_of_line = crlf 14 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | graft docs 2 | graft src 3 | graft ci 4 | graft tests 5 | 6 | include .bumpversion.cfg 7 | 8 | include AUTHORS.rst 9 | include CHANGELOG.rst 10 | include CONTRIBUTING.rst 11 | include LICENSE 12 | include README.rst 13 | include EXAMPLES.rst 14 | 15 | global-exclude *.py[cod] __pycache__/* *.so *.dylib 16 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | ======== 2 | Contents 3 | ======== 4 | 5 | .. toctree:: 6 | :maxdepth: 2 7 | 8 | readme 9 | reference/index 10 | contributing 11 | authors 12 | changelog 13 | 14 | Indices and tables 15 | ================== 16 | 17 | * :ref:`genindex` 18 | * :ref:`modindex` 19 | * :ref:`search` 20 | 21 | -------------------------------------------------------------------------------- /src/humiolib/__main__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Entrypoint module, in case you use `python -m humiolib`. 3 | 4 | 5 | Why does this file exist, and why __main__? For more info, read: 6 | 7 | - https://www.python.org/dev/peps/pep-0338/ 8 | - https://docs.python.org/2/using/cmdline.html#cmdoption-m 9 | - https://docs.python.org/3/using/cmdline.html#cmdoption-m 10 | """ 11 | from humiolib.cli import main 12 | 13 | if __name__ == "__main__": 14 | main() 15 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | # 2 | 3 | Link To Issue Being Solved: **_#ADD LINK TO ISSUE_** 4 | 5 | **What Changes Have Been Made?** 6 | 7 | Describe what changes your contribution introduces to the code base. 8 | 9 | **Why Have the Changes Been Made?** 10 | 11 | Describe why these changes have been introduced. 12 | 13 | **How Do These Changes Impact the User?** 14 | 15 | Describe how the changes will impact the user? Are the changes breaking and require them to put in some work, when they update the project? 16 | -------------------------------------------------------------------------------- /src/humiolib/HumioExceptions.py: -------------------------------------------------------------------------------- 1 | class HumioException(Exception): 2 | pass 3 | 4 | class HumioConnectionException(HumioException): 5 | pass 6 | 7 | class HumioHTTPException(HumioException): 8 | def __init__(self, message, status_code=None): 9 | self.message = message 10 | self.status_code = status_code 11 | 12 | class HumioTimeoutException(HumioException): 13 | pass 14 | 15 | class HumioConnectionDroppedException(HumioException): 16 | pass 17 | 18 | class HumioQueryJobExhaustedException(HumioException): 19 | pass 20 | 21 | class HumioQueryJobExpiredException(HumioException): 22 | pass -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | Authors 2 | ======= 3 | 4 | Current Maintainer(s) 5 | ********************* 6 | 7 | * Alexander Brandborg, `@alexanderbrandborg `_ 8 | 9 | Original Author and First Commit 10 | ******************************** 11 | 12 | * Sergey Grigorev, `@xorsnn `_ 13 | 14 | Contributors (alpha by username) 15 | ******************************** 16 | 17 | * Anders Fogh Eriksen `@Fogh `_ 18 | * Chris Fraser `@swefraser `_ 19 | * Hanne Moa `@hmpf `_ 20 | * Kristian Gausel `@KGausel `_ 21 | * Mo Latif `@molatif-def ` 22 | * Peter Mechlenborg `@pmech `_ 23 | * Sam `@samgdf `_ 24 | * Vishal Kuo `@vishalkuo `_ 25 | 26 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 1 3 | 4 | [flake8] 5 | max-line-length = 140 6 | exclude = */migrations/* 7 | 8 | [tool:pytest] 9 | # If a pytest section is found in one of the possible config files 10 | # (pytest.ini, tox.ini or setup.cfg), then pytest will not look for any others, 11 | # so if you add a pytest config section elsewhere, 12 | # you will need to delete this section from setup.cfg. 13 | norecursedirs = 14 | migrations 15 | 16 | python_files = 17 | test_*.py 18 | *_test.py 19 | tests.py 20 | addopts = 21 | -ra 22 | --strict 23 | --doctest-modules 24 | --doctest-glob=\*.rst 25 | --tb=short 26 | testpaths = 27 | tests 28 | 29 | [tool:isort] 30 | force_single_line = True 31 | line_length = 120 32 | known_first_party = humiolib 33 | default_section = THIRDPARTY 34 | forced_separate = test_humiolib 35 | not_skip = __init__.py 36 | skip = migrations 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | __pycache__ 3 | .pypirc 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Packages 9 | *.egg 10 | *.egg-info 11 | dist 12 | build 13 | eggs 14 | .eggs 15 | parts 16 | bin 17 | var 18 | sdist 19 | wheelhouse 20 | develop-eggs 21 | .installed.cfg 22 | lib 23 | lib64 24 | venv*/ 25 | pyvenv*/ 26 | pip-wheel-metadata/ 27 | 28 | # Installer logs 29 | pip-log.txt 30 | 31 | # Unit test / coverage reports 32 | .coverage 33 | .tox 34 | .coverage.* 35 | .pytest_cache/ 36 | nosetests.xml 37 | coverage.xml 38 | htmlcov 39 | 40 | # Translations 41 | *.mo 42 | 43 | # Mr Developer 44 | .mr.developer.cfg 45 | .project 46 | .pydevproject 47 | .idea 48 | *.iml 49 | *.komodoproject 50 | 51 | # Complexity 52 | output/*.html 53 | output/*/index.html 54 | 55 | # Sphinx 56 | docs/_build 57 | 58 | .DS_Store 59 | *~ 60 | .*.sw[po] 61 | .build 62 | .ve 63 | .env 64 | .cache 65 | .pytest 66 | .benchmarks 67 | .bootstrap 68 | .appveyor.token 69 | *.bak 70 | 71 | # Mypy Cache 72 | .mypy_cache/ 73 | 74 | .vscode/ -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | import os 5 | 6 | extensions = [ 7 | "sphinx.ext.autodoc", 8 | "sphinx.ext.autosummary", 9 | "sphinx.ext.coverage", 10 | "sphinx.ext.doctest", 11 | "sphinx.ext.extlinks", 12 | "sphinx.ext.ifconfig", 13 | "sphinx.ext.napoleon", 14 | "sphinx.ext.todo", 15 | "sphinx.ext.viewcode", 16 | ] 17 | source_suffix = ".rst" 18 | master_doc = "index" 19 | project = "humiolib" 20 | year = "2020" 21 | author = "Humio ApS" 22 | copyright = "{0}, {1}".format(year, author) 23 | version = "0.2.6" 24 | 25 | pygments_style = "trac" 26 | templates_path = ["."] 27 | extlinks = { 28 | "issue": ("https://github.com/humio/python-humio/issues/%s", "#"), 29 | "pr": ("https://github.com/humio/python-humio/pulls/%s", "PR #"), 30 | } 31 | 32 | 33 | import sphinx_rtd_theme 34 | 35 | extensions = [ 36 | "sphinx_rtd_theme", 37 | "sphinx.ext.autodoc" 38 | ] 39 | 40 | html_theme = "sphinx_rtd_theme" 41 | -------------------------------------------------------------------------------- /tests/cassettes/humioclient/test_get_users: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: null 4 | headers: 5 | Accept: 6 | - '*/*' 7 | Accept-Encoding: 8 | - gzip, deflate 9 | Connection: 10 | - keep-alive 11 | Content-Type: 12 | - application/json 13 | User-Agent: 14 | - python-requests/2.22.0 15 | method: GET 16 | uri: https://cloud.humio.com/api/v1/users 17 | response: 18 | body: 19 | string: !!binary | 20 | H4sIAAAAAAAAABzKwQnAQAhE0VamqDQgRlgh6KLjJdVnyfW/fy1Dz96P2w0ZLgu6Cj0D3ojkX7P8 21 | PQMTomrd4Dpa1jml9gEAAP//AwBkvPpgRQAAAA== 22 | headers: 23 | Connection: 24 | - keep-alive 25 | Content-Encoding: 26 | - gzip 27 | Content-Type: 28 | - text/plain; charset=UTF-8 29 | Date: 30 | - Wed, 25 Mar 2020 14:47:34 GMT 31 | Server: 32 | - nginx 33 | Transfer-Encoding: 34 | - chunked 35 | status: 36 | code: 403 37 | message: Forbidden 38 | version: 1 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /tests/cassettes/humioclient/test_create_queryjob_incorrect_query_syntax: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"queryString": "timechart(func=nowork)"}' 4 | headers: 5 | Accept: 6 | - '*/*' 7 | Accept-Encoding: 8 | - gzip, deflate 9 | Connection: 10 | - keep-alive 11 | Content-Length: 12 | - '41' 13 | Content-Type: 14 | - application/json 15 | User-Agent: 16 | - python-requests/2.22.0 17 | method: POST 18 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs 19 | response: 20 | body: 21 | string: !!binary | 22 | H4sIAAAAAAAAACpILErMTS1JLVJIK81LVsgsVsjLL1EoLi0oyC8qSU1RyMwDS5Rk5ucplGTmpjpn 23 | JBaVcCUkJHCBCAAAAAD//wMAfinKTj4AAAA= 24 | headers: 25 | Connection: 26 | - keep-alive 27 | Content-Encoding: 28 | - gzip 29 | Content-Type: 30 | - text/plain; charset=UTF-8 31 | Date: 32 | - Wed, 25 Mar 2020 14:12:46 GMT 33 | Server: 34 | - nginx 35 | Transfer-Encoding: 36 | - chunked 37 | status: 38 | code: 400 39 | message: Bad Request 40 | version: 1 41 | -------------------------------------------------------------------------------- /tests/cassettes/humioclient/test_get_status: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: null 4 | headers: 5 | Accept: 6 | - '*/*' 7 | Accept-Encoding: 8 | - gzip, deflate 9 | Connection: 10 | - keep-alive 11 | User-Agent: 12 | - python-requests/2.22.0 13 | method: GET 14 | uri: https://cloud.humio.com/api/v1/status 15 | response: 16 | body: 17 | string: !!binary | 18 | H4sIAAAAAAAAAKpWKi5JLCktVrJSUPL3VtJRUCpLLSrOzM8DCRjqWeoZ6eomlWbmpOgaGhmbmerq 19 | Fmck6ialmRhaWJhYGCUq1QIAAAD//wMAZUTSWEEAAAA= 20 | headers: 21 | Connection: 22 | - keep-alive 23 | Content-Encoding: 24 | - gzip 25 | Content-Type: 26 | - application/json 27 | Date: 28 | - Wed, 25 Mar 2020 14:42:29 GMT 29 | Server: 30 | - nginx 31 | Strict-Transport-Security: 32 | - max-age=31536000;includeSubDomains 33 | Transfer-Encoding: 34 | - chunked 35 | X-Content-Type-Options: 36 | - nosniff 37 | X-XSS-Protection: 38 | - 1;mode=block 39 | status: 40 | code: 200 41 | message: OK 42 | version: 1 43 | -------------------------------------------------------------------------------- /.github/workflows/pythonapp.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a single version of Python 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: Python application 5 | 6 | on: 7 | push: 8 | branches: [ master ] 9 | pull_request: 10 | branches: [ master ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | - name: Set up Python 3.8 20 | uses: actions/setup-python@v1 21 | with: 22 | python-version: 3.8 23 | - name: Install dependencies 24 | run: | 25 | python -m pip install --upgrade pip 26 | pip install -e . 27 | - name: Lint with flake8 28 | run: | 29 | pip install flake8 30 | # stop the build if there are Python syntax errors or undefined names 31 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 32 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 33 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 34 | - name: Test with pytest 35 | run: | 36 | pip install pytest 37 | pytest 38 | -------------------------------------------------------------------------------- /tests/cassettes/humioclient/test_create_queryjob_success: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"queryString": "timechart()"}' 4 | headers: 5 | Accept: 6 | - '*/*' 7 | Accept-Encoding: 8 | - gzip, deflate 9 | Connection: 10 | - keep-alive 11 | Content-Length: 12 | - '30' 13 | Content-Type: 14 | - application/json 15 | User-Agent: 16 | - python-requests/2.22.0 17 | method: POST 18 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs 19 | response: 20 | body: 21 | string: !!binary | 22 | H4sIAAAAAAAAAKpWykxRslIyMtH1CcmwLDcwCzIqyPQrC/TwzMzINPUp94tS0lEqLE0tqvTPC8tM 23 | LQeqLcnMTU3OSCwq0dBUqgUAAAD//wMAlrWCwkAAAAA= 24 | headers: 25 | Connection: 26 | - keep-alive 27 | Content-Encoding: 28 | - gzip 29 | Content-Type: 30 | - application/json 31 | Date: 32 | - Wed, 25 Mar 2020 14:12:45 GMT 33 | Server: 34 | - nginx 35 | Strict-Transport-Security: 36 | - max-age=31536000;includeSubDomains 37 | Transfer-Encoding: 38 | - chunked 39 | X-Content-Type-Options: 40 | - nosniff 41 | X-XSS-Protection: 42 | - 1;mode=block 43 | status: 44 | code: 200 45 | message: OK 46 | version: 1 47 | -------------------------------------------------------------------------------- /tests/cassettes/humioclient/test_ingest_json_on_humioclient_success: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '[{"tags": {"host": "server1", "source": "application.log"}, "events": [{"timestamp": 4 | "2020-03-23T00:00:00+00:00", "attributes": {"key1": "value1", "key2": "value2"}}]}]' 5 | headers: 6 | Accept: 7 | - '*/*' 8 | Accept-Encoding: 9 | - gzip, deflate 10 | Connection: 11 | - keep-alive 12 | Content-Length: 13 | - '168' 14 | Content-Type: 15 | - application/json 16 | User-Agent: 17 | - python-requests/2.22.0 18 | method: POST 19 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/ingest 20 | response: 21 | body: 22 | string: !!binary | 23 | H4sIAAAAAAAAAKquBQAAAP//AwBDv6ajAgAAAA== 24 | headers: 25 | Connection: 26 | - keep-alive 27 | Content-Encoding: 28 | - gzip 29 | Content-Type: 30 | - application/json 31 | Date: 32 | - Wed, 25 Mar 2020 14:12:49 GMT 33 | Server: 34 | - nginx 35 | Strict-Transport-Security: 36 | - max-age=31536000;includeSubDomains 37 | Transfer-Encoding: 38 | - chunked 39 | X-Content-Type-Options: 40 | - nosniff 41 | X-XSS-Protection: 42 | - 1;mode=block 43 | status: 44 | code: 200 45 | message: OK 46 | version: 1 47 | -------------------------------------------------------------------------------- /tests/cassettes/humioclient/test_ingest_json_on_ingestclient_success: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '[{"tags": {"host": "server1", "source": "application.log"}, "events": [{"timestamp": 4 | "2020-03-23T00:00:00+00:00", "attributes": {"key1": "value1", "key2": "value2"}}]}]' 5 | headers: 6 | Accept: 7 | - '*/*' 8 | Accept-Encoding: 9 | - gzip, deflate 10 | Connection: 11 | - keep-alive 12 | Content-Length: 13 | - '168' 14 | Content-Type: 15 | - application/json 16 | User-Agent: 17 | - python-requests/2.22.0 18 | method: POST 19 | uri: https://cloud.humio.com/api/v1/ingest/humio-structured 20 | response: 21 | body: 22 | string: !!binary | 23 | H4sIAAAAAAAAAKquBQAAAP//AwBDv6ajAgAAAA== 24 | headers: 25 | Connection: 26 | - keep-alive 27 | Content-Encoding: 28 | - gzip 29 | Content-Type: 30 | - application/json 31 | Date: 32 | - Wed, 25 Mar 2020 14:12:49 GMT 33 | Server: 34 | - nginx 35 | Strict-Transport-Security: 36 | - max-age=31536000;includeSubDomains 37 | Transfer-Encoding: 38 | - chunked 39 | X-Content-Type-Options: 40 | - nosniff 41 | X-XSS-Protection: 42 | - 1;mode=block 43 | status: 44 | code: 200 45 | message: OK 46 | version: 1 47 | -------------------------------------------------------------------------------- /tests/cassettes/humioclient/test_ingest_messages_on_ingestclient_ssuccess: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '[{"messages": ["192.168.1.49 - user1 [02/Nov/2017:13:48:33 +0000] \"POST 4 | /humio/api/v1/ingest/elastic-bulk HTTP/1.1\" 200 0 \"-\" \"useragent\" 0.014 5 | 657 0.014", "192.168.1..21 - user2 [02/Nov/2017:13:49:09 +0000] \"POST /humio/api/v1/ingest/elastic-bulk 6 | HTTP/1.1\" 200 0 \"-\" \"useragent\" 0.013 565 0.013"]}]' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '311' 16 | Content-Type: 17 | - application/json 18 | User-Agent: 19 | - python-requests/2.22.0 20 | method: POST 21 | uri: https://cloud.humio.com/api/v1/ingest/humio-unstructured 22 | response: 23 | body: 24 | string: !!binary | 25 | H4sIAAAAAAAAAKquBQAAAP//AwBDv6ajAgAAAA== 26 | headers: 27 | Connection: 28 | - keep-alive 29 | Content-Encoding: 30 | - gzip 31 | Content-Type: 32 | - application/json 33 | Date: 34 | - Wed, 25 Mar 2020 14:12:49 GMT 35 | Server: 36 | - nginx 37 | Strict-Transport-Security: 38 | - max-age=31536000;includeSubDomains 39 | Transfer-Encoding: 40 | - chunked 41 | X-Content-Type-Options: 42 | - nosniff 43 | X-XSS-Protection: 44 | - 1;mode=block 45 | status: 46 | code: 200 47 | message: OK 48 | version: 1 49 | -------------------------------------------------------------------------------- /tests/cassettes/humioclient/test_ingest_messages_on_humioclient_success: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '[{"messages": ["192.168.1.49 - user1 [02/Nov/2017:13:48:33 +0000] \"POST 4 | /humio/api/v1/ingest/elastic-bulk HTTP/1.1\" 200 0 \"-\" \"useragent\" 0.014 5 | 657 0.014", "192.168.1..21 - user2 [02/Nov/2017:13:49:09 +0000] \"POST /humio/api/v1/ingest/elastic-bulk 6 | HTTP/1.1\" 200 0 \"-\" \"useragent\" 0.013 565 0.013"]}]' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '311' 16 | Content-Type: 17 | - application/json 18 | User-Agent: 19 | - python-requests/2.22.0 20 | method: POST 21 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/ingest-messages 22 | response: 23 | body: 24 | string: !!binary | 25 | H4sIAAAAAAAAAKquBQAAAP//AwBDv6ajAgAAAA== 26 | headers: 27 | Connection: 28 | - keep-alive 29 | Content-Encoding: 30 | - gzip 31 | Content-Type: 32 | - application/json 33 | Date: 34 | - Wed, 25 Mar 2020 14:12:48 GMT 35 | Server: 36 | - nginx 37 | Strict-Transport-Security: 38 | - max-age=31536000;includeSubDomains 39 | Transfer-Encoding: 40 | - chunked 41 | X-Content-Type-Options: 42 | - nosniff 43 | X-XSS-Protection: 44 | - 1;mode=block 45 | status: 46 | code: 200 47 | message: OK 48 | version: 1 49 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | 2 | Changelog 3 | ========= 4 | 5 | 0.2.0 (2020-03-30) 6 | ****************** 7 | Initial real release to PyPI 8 | 9 | Added: 10 | 11 | * Tests, mocking out API calls with vcr.py 12 | * Custom error handling to completly wrap url library used 13 | * QueryJob class 14 | 15 | Changed: 16 | 17 | * Whole API interface has been updated 18 | * Updated Sphinx documentation 19 | 20 | Removed: 21 | 22 | * A few configuration files left over from earlier versions 23 | 24 | 25 | 0.2.2 (2020-05-19) 26 | ****************** 27 | Bugfixing to ensure that static queryjobs can be polled for all their results 28 | 29 | Added: 30 | 31 | * Static queryjobs can now be queried for more than one segment 32 | 33 | 34 | Changed: 35 | 36 | * Upon polling from a QueryJob it will now stall until it can poll data from Humio, ensuring that an empty result is not returned prematurely. 37 | 38 | Removed: 39 | 40 | * The poll_until_done method has been removed from live query jobs, as this does not make conceptual sense to do, in the same manner as a static query job. 41 | 42 | 0.2.3 (2021-08-13) 43 | ****************** 44 | Smaller bugfixes 45 | Changed: 46 | 47 | * Fix urls in docstrings in HumioClient.py 48 | * Propagate kwargs to poll functions in QueryJob.py 49 | 50 | 0.2.4 (2022-08-15) 51 | ****************** 52 | Smaller file related bugfixes 53 | Changed: 54 | 55 | * upload_file function no longer attempts a cast to json 56 | * list_files function now works on newer versions of humio 57 | 58 | 0.2.5 (2023-04-17) 59 | ****************** 60 | Expand file functionality 61 | Changed: 62 | 63 | * Added additional endpoints for manipulating files via GraphQL 64 | 65 | 66 | 0.2.6 (2023-10-05) 67 | ****************** 68 | Add 'Saved Query' Support 69 | Changed: 70 | 71 | * Added additional endpoints for managing saved queries -------------------------------------------------------------------------------- /tests/cassettes/humioclient/test_streaming_query_success: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"queryString": "timechart()"}' 4 | headers: 5 | Accept: 6 | - application/x-ndjson 7 | Accept-Encoding: 8 | - gzip, deflate 9 | Connection: 10 | - keep-alive 11 | Content-Length: 12 | - '30' 13 | Content-Type: 14 | - application/json 15 | User-Agent: 16 | - python-requests/2.22.0 17 | method: POST 18 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/query 19 | response: 20 | body: 21 | string: !!binary | 22 | H4sIAAAAAAAAAIzXPWoDMRCG4T7H2DrF/EiamVzGEJMqkDRxZXL3rPESCBbRq27hY9Dq0Ujoup3O 23 | n5ePr+1lk+15O71ezu9vty/t2aVnk9vYvp+u/wfLWXCIwaAqDN4Lruc4rGBFTxhsAYN9wODoMBhQ 24 | ZiSVKSgTAmXinlvLhEKZMCgTDmWiQZnoUCYGlImAMpFUpqDMfRVBX6dAmVQokwZl0qFMNiiTHcrk 25 | gDIZUCaTytzXe90zWVCmBMqUQpkyKFMOZapBmepQpgaUqeOUWl5xdTQNCFKZo+Sq4r51mIyKMhkV 26 | YzIqzmRUGpNR6UxGZbCe0WN1lj2zB5mMSrKeUSkos/8NuthVFcqoQRl1KKMNymiHMsd2XMvogDIa 27 | UEaTyhSUMYEyplDGDMqYQxlrUObo/7WMdShjA8pYQBlLKlNQxgXKuEIZNyjjDmWOA3ct4w3KeIcy 28 | PqCMx0zGHh6Q6vsj8jb+XnE6S9aMpj2WbDKjGZOgzmjs8Zm7n3szm9ksj0tujdN8hjP7nfaL8wMA 29 | AP//AwDquN/fjA8AAA== 30 | headers: 31 | Connection: 32 | - keep-alive 33 | Content-Encoding: 34 | - gzip 35 | Content-Type: 36 | - application/x-ndjson 37 | Date: 38 | - Wed, 25 Mar 2020 14:12:46 GMT 39 | Server: 40 | - nginx 41 | Strict-Transport-Security: 42 | - max-age=31536000;includeSubDomains 43 | Transfer-Encoding: 44 | - chunked 45 | X-Content-Type-Options: 46 | - nosniff 47 | X-XSS-Protection: 48 | - 1;mode=block 49 | humio-query-id: 50 | - IQ-QsEVJQpWp9Tbh8nfvFze2X8J 51 | status: 52 | code: 200 53 | message: OK 54 | version: 1 55 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | from __future__ import absolute_import 4 | from __future__ import print_function 5 | 6 | import io 7 | import re 8 | from glob import glob 9 | from os.path import basename 10 | from os.path import dirname 11 | from os.path import join 12 | from os.path import splitext 13 | 14 | from setuptools import find_packages 15 | from setuptools import setup 16 | 17 | def read(*names, **kwargs): 18 | with io.open( 19 | join(dirname(__file__), *names), encoding=kwargs.get("encoding", "utf8") 20 | ) as fh: 21 | return fh.read() 22 | 23 | setup( 24 | name="humiolib", 25 | version="0.2.6", 26 | license="Apache-2.0", 27 | description="Python SDK for connecting to Humio", 28 | long_description="%s\n%s" 29 | % ( 30 | re.compile("^.. start-badges.*^.. end-badges", re.M | re.S).sub( 31 | "", read("README.rst") 32 | ), 33 | re.sub(":[a-z]+:`~?(.*?)`", r"``\1``", read("CHANGELOG.rst")), 34 | ), 35 | long_description_content_type = "text/x-rst", 36 | author="Humio ApS", 37 | author_email="integrations@humio.com", 38 | url="https://github.com/humio/python-humio", 39 | packages=find_packages("src"), 40 | package_dir={"": "src"}, 41 | py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")], 42 | include_package_data=True, 43 | zip_safe=False, 44 | classifiers=[ 45 | # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers 46 | "Development Status :: 5 - Production/Stable", 47 | "Intended Audience :: Developers", 48 | "License :: OSI Approved :: Apache Software License", 49 | "Operating System :: Unix", 50 | "Operating System :: POSIX", 51 | "Operating System :: Microsoft :: Windows", 52 | "Programming Language :: Python", 53 | "Programming Language :: Python :: 3", 54 | "Programming Language :: Python :: 3.5", 55 | "Programming Language :: Python :: 3.6", 56 | "Programming Language :: Python :: 3.7", 57 | "Programming Language :: Python :: 3.8", 58 | "Programming Language :: Python :: Implementation :: CPython", 59 | "Programming Language :: Python :: Implementation :: PyPy", 60 | # uncomment if you test on these interpreters: 61 | # 'Programming Language :: Python :: Implementation :: IronPython', 62 | # 'Programming Language :: Python :: Implementation :: Jython', 63 | # 'Programming Language :: Python :: Implementation :: Stackless', 64 | "Topic :: System :: Logging", 65 | "Topic :: System :: Monitoring", 66 | ], 67 | project_urls={ 68 | "Documentation": "https://python-humio.readthedocs.io/", 69 | "Changelog": "https://python-humio.readthedocs.io/en/latest/changelog.html", 70 | "Issue Tracker": "https://github.com/humio/python-humio/issues", 71 | }, 72 | keywords=["humio", "log management"], 73 | python_requires=">=3.5", 74 | install_requires=[ 75 | "requests", 76 | "pytest", 77 | "vcrpy" 78 | ], 79 | extras_require={ 80 | # eg: 81 | # 'rst': ['docutils>=0.11'], 82 | # ':python_version=="2.6"': ['argparse'], 83 | }, 84 | entry_points={"console_scripts": ["humiocli = humiolib.cli:main"]}, 85 | ) 86 | -------------------------------------------------------------------------------- /tests/test_queryjob.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import vcr 3 | import os 4 | from humiolib import HumioClient 5 | from humiolib.HumioExceptions import HumioQueryJobExhaustedException 6 | 7 | user_token = os.environ['HUMIO_USER_TOKEN'] if 'HUMIO_USER_TOKEN' in os.environ else "bogustoken" 8 | ingest_token = os.environ['HUMIO_INGEST_TOKEN'] if 'HUMIO_INGEST_TOKEN' in os.environ else "bogustoken" 9 | 10 | dirname = os.path.dirname(__file__) 11 | cassettedir = os.path.join(dirname, 'cassettes/queryjob/') 12 | 13 | @pytest.fixture 14 | def humioclient(): 15 | client = HumioClient( 16 | base_url= "https://cloud.humio.com", 17 | repository= "sandbox", 18 | user_token=user_token, 19 | ) 20 | return client 21 | 22 | # STATIC QUERY JOB TESTS 23 | @vcr.use_cassette(cassette_library_dir=cassettedir, filter_headers=['Authorization']) 24 | def test_poll_until_done_static_queryjob_aggregate_query(humioclient): 25 | queryjob = humioclient.create_queryjob("timechart()", is_live=False) 26 | 27 | events = [] 28 | for pollResult in queryjob.poll_until_done(): 29 | events.extend(pollResult.events) 30 | 31 | assert len(events) != 0 32 | 33 | 34 | @vcr.use_cassette(cassette_library_dir=cassettedir, filter_headers=['Authorization']) 35 | def test_poll_until_done_static_queryjob_non_aggregate_query(humioclient): 36 | queryjob = humioclient.create_queryjob("", is_live=False) 37 | 38 | events = [] 39 | for pollResult in queryjob.poll_until_done(): 40 | events.extend(pollResult.events) 41 | 42 | assert len(events) != 0 43 | 44 | 45 | @vcr.use_cassette(cassette_library_dir=cassettedir, filter_headers=['Authorization']) 46 | def test_poll_until_done_static_queryjob_after_fully_polled_fail(humioclient): 47 | queryjob = humioclient.create_queryjob("timechart()", is_live=False) 48 | 49 | events = [] 50 | for pollResult in queryjob.poll_until_done(): 51 | events.extend(pollResult.events) 52 | 53 | with pytest.raises(HumioQueryJobExhaustedException): 54 | for pollResult in queryjob.poll_until_done(): 55 | events.extend(pollResult.events) 56 | 57 | 58 | # LIVE QUERY JOB TESTS 59 | @vcr.use_cassette(cassette_library_dir=cassettedir, filter_headers=['Authorization']) 60 | def test_poll_until_done_live_queryjob_aggregate_query(humioclient): 61 | queryjob = humioclient.create_queryjob("timechart()", is_live=True) 62 | 63 | events = [] 64 | for poll_events in queryjob.poll().events: 65 | events.extend(poll_events) 66 | 67 | assert len(events) != 0 68 | 69 | @vcr.use_cassette(cassette_library_dir=cassettedir, filter_headers=['Authorization']) 70 | def test_poll_until_done_live_queryjob_non_aggregate_query(humioclient): 71 | queryjob = humioclient.create_queryjob("", is_live=True) 72 | 73 | events = [] 74 | for poll_events in queryjob.poll().events: 75 | events.extend(poll_events) 76 | 77 | assert len(events) != 0 78 | 79 | @vcr.use_cassette(cassette_library_dir=cassettedir, filter_headers=['Authorization']) 80 | def test_poll_until_done_live_queryjob_poll_after_done_success(humioclient): 81 | queryjob = humioclient.create_queryjob("timechart()", is_live=True) 82 | 83 | first_poll_events = [] 84 | for poll_events in queryjob.poll().events: 85 | first_poll_events.extend(poll_events) 86 | 87 | second_poll_events = [] 88 | for poll_events in queryjob.poll().events: 89 | second_poll_events.extend(poll_events) 90 | 91 | assert len(second_poll_events) != 0 -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at integrations@humio.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /tests/cassettes/queryjob/test_poll_until_done_static_queryjob_aggregate_query: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"queryString": "timechart()", "isLive": false}' 4 | headers: 5 | Accept: 6 | - '*/*' 7 | Accept-Encoding: 8 | - gzip, deflate 9 | Connection: 10 | - keep-alive 11 | Content-Length: 12 | - '47' 13 | Content-Type: 14 | - application/json 15 | User-Agent: 16 | - python-requests/2.22.0 17 | method: POST 18 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs 19 | response: 20 | body: 21 | string: !!binary | 22 | H4sIAAAAAAAAAKpWykxRslIyMtGtDPf0Ncr3dLLwriwr9vfNcTfIdnMtjUpT0lEqLE0tqvTPC8tM 23 | LQeqLcnMTU3OSCwq0dBUqgUAAAD//wMA+rinBkAAAAA= 24 | headers: 25 | Connection: 26 | - keep-alive 27 | Content-Encoding: 28 | - gzip 29 | Content-Type: 30 | - application/json 31 | Date: 32 | - Wed, 25 Mar 2020 14:33:12 GMT 33 | Server: 34 | - nginx 35 | Strict-Transport-Security: 36 | - max-age=31536000;includeSubDomains 37 | Transfer-Encoding: 38 | - chunked 39 | X-Content-Type-Options: 40 | - nosniff 41 | X-XSS-Protection: 42 | - 1;mode=block 43 | status: 44 | code: 200 45 | message: OK 46 | - request: 47 | body: null 48 | headers: 49 | Accept: 50 | - '*/*' 51 | Accept-Encoding: 52 | - gzip, deflate 53 | Connection: 54 | - keep-alive 55 | Content-Type: 56 | - application/json 57 | User-Agent: 58 | - python-requests/2.22.0 59 | method: GET 60 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs/24-yWIM2oIB8KyvsOMlG0kFEuZf 61 | response: 62 | body: 63 | string: !!binary | 64 | H4sIAAAAAAAAAIzYz2+bMBQH8H9l8nGikp9tsM2tXXqYtB22atphmiJKnBSVQAem2Vb1f58Jbtqu 65 | SHxzyo8n+9kfDHnvgZVFU7q6dhuWb4u6dwnbtI1jue+G8N7du8b3LP/xwNZlOzSe5YyzhK2vh/LW 66 | jZ8oNSnPuODjiz0mC4FEYOA0IDCisOCI0oCBSoOBaQYGZikYqBUYaCQYaEEZzUEZPcUty2gCZbQA 67 | ZbQEZbQCZXQKyugMlNEalNEGlbGgzLSLwCk0HJQxBMoYAcoYCcoYBcqYFJQxGShjNChjDCoz7ffy 68 | mTEWlLEclLEEylgBylgJylgFytgUlLEZKGPjXWrxgWTjoQECUZk45NKI4dLBZIgTJkNcYDLEJSZD 69 | XGEyxFNMhniGnRmKu7N4ZkIgJkPcYGeGuAVlwmqgxzARgTIkQBmSoAwpUIZSUCZejssylIEypEEZ 70 | MqiMBWUEB2UEgTJCgDJCgjJCgTLx/C/LiBSUERkoIzQoIwwqY0EZyUEZSaCMFKCMlKBMvOEuy0gF 71 | ysgUlJEZKCP1nIx4U+6RDCXf+Hr9iKO5SDtHo94OqfgcTTYTSHM04m1RGu57czZzWcaH3DKOknM4 72 | ZiZLBeKoFMRR2RPOz4TtnS9WhS9Y/jCV6B+m2tzqULL/9t3pR1/tXXlTdGOlPlb0IdOpcl/XRe9f 73 | V/F0muIU1N8Vzfpm2BdN9XfsEIRq/92+agbv+ueRjkH7qq6r0ChgcYMSNlR5P+x2rvduc3aoNrtj 74 | u2BM6GzK6DTLtur+z+VlR4Ftq9q77svguj/jgn+Nb658VzW7MOD7kEjVf2xCRFH66j60LmIfo2rK 75 | eti4latdSOEydjLij64Jyxl7F2HV2gphbML6m/ZwnOUYu6r6MMf14Ku2eR6z//Ryit4ftzb2QGQc 76 | p2m/un6o/bfGV/Xq2E05zvo4Znq+23VuV/hTi+WurevzbUg/jGO1CYJ3XVu6vnebiz/jRufhjwtl 77 | L75+WkqmkmkzLmcWE3dpLr/umN3FsN267irAsny8bkaZz1FRyfC59UX9ve1uWS4Sdii6Jmz42AoK 78 | 198hfD0tTDw+/gMAAP//AwDuFXSIRBIAAA== 79 | headers: 80 | Connection: 81 | - keep-alive 82 | Content-Encoding: 83 | - gzip 84 | Content-Type: 85 | - application/json 86 | Date: 87 | - Wed, 25 Mar 2020 14:33:12 GMT 88 | Server: 89 | - nginx 90 | Strict-Transport-Security: 91 | - max-age=31536000;includeSubDomains 92 | Transfer-Encoding: 93 | - chunked 94 | X-Content-Type-Options: 95 | - nosniff 96 | X-XSS-Protection: 97 | - 1;mode=block 98 | status: 99 | code: 200 100 | message: OK 101 | version: 1 102 | -------------------------------------------------------------------------------- /tests/cassettes/queryjob/test_poll_until_done_static_queryjob_after_fully_polled_fail: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"queryString": "timechart()", "isLive": false}' 4 | headers: 5 | Accept: 6 | - '*/*' 7 | Accept-Encoding: 8 | - gzip, deflate 9 | Connection: 10 | - keep-alive 11 | Content-Length: 12 | - '47' 13 | Content-Type: 14 | - application/json 15 | User-Agent: 16 | - python-requests/2.22.0 17 | method: POST 18 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs 19 | response: 20 | body: 21 | string: !!binary | 22 | H4sIAAAAAAAAAKpWykxRslIyMtEt9irz9M32MMkr9vPzNI0ySwnOMfPwdPVT0lEqLE0tqvTPC8tM 23 | LQeqLcnMTU3OSCwq0dBUqgUAAAD//wMA7xhZ5kAAAAA= 24 | headers: 25 | Connection: 26 | - keep-alive 27 | Content-Encoding: 28 | - gzip 29 | Content-Type: 30 | - application/json 31 | Date: 32 | - Wed, 25 Mar 2020 14:24:21 GMT 33 | Server: 34 | - nginx 35 | Strict-Transport-Security: 36 | - max-age=31536000;includeSubDomains 37 | Transfer-Encoding: 38 | - chunked 39 | X-Content-Type-Options: 40 | - nosniff 41 | X-XSS-Protection: 42 | - 1;mode=block 43 | status: 44 | code: 200 45 | message: OK 46 | - request: 47 | body: null 48 | headers: 49 | Accept: 50 | - '*/*' 51 | Accept-Encoding: 52 | - gzip, deflate 53 | Connection: 54 | - keep-alive 55 | Content-Type: 56 | - application/json 57 | User-Agent: 58 | - python-requests/2.22.0 59 | method: GET 60 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs/24-sJvIMkH4nsNNI5Z6dSl6HIEN 61 | response: 62 | body: 63 | string: !!binary | 64 | H4sIAAAAAAAAAIzYXW+bMBQG4L8y+XKiko9tsM1du/Ri0naxVdMupimixElRCXRgmnVV/vvMR9N2 65 | RePNVT6O7GM/GHLOI8uzKndl6TYs3WZl6yK2qSvHUt904b27d5VvWfrjka3zuqs8SxlnEVtfd/mt 66 | 6z9RbGIeW8n7FztG/w9MuAADicDAcUBgamHBEaUBA5UGA+MEDExiMFArMNCgMhaU0RyU0WPcsowm 67 | UEYLUEZLUEYrUEbHoIxOQBmtQRltUBkLyoy7CBxXw0EZQ6CMEaCMkaCMUaCMiUEZk4AyRoMyxqAy 68 | 434vnxljQRnLQRlLoIwVoIyVoIxVoIyNQRmbgDJ2ukstPrnsdGiAQFRmGnJpxHDpYDLECZMhLjAZ 69 | 4hKTIa4wGeIxJkM8wc4MTbuzeGZCICZD3GBnhrgFZcJqoOc1EYEyJEAZkqAMKVCGYlBmuhyXZSgB 70 | ZUiDMmRQGQvKCA7KCAJlhABlhARlhAJlpvO/LCNiUEYkoIzQoIwwqIwFZSQHZSSBMlKAMlKCMtMN 71 | d1lGKlBGxqCMTEAZqedkxJu6kGSoDfvX60cczUXaORr1dkjF52iSmUCaoxFvq9dw35uzmctyesgt 72 | 4yg5h2NmslQgjoqfcH5GbO98tsp8xtLHsUT/MNbmVoeS/bdvTj/6Yu/ym6zpK/W+og8JjJX7usxa 73 | /7qKp9MUp6D2LqvWN90+q4o/fYcgVPvv9kXVedc+jzQE7YuyLEKjgE3rjlhXpG2327nWu83Zodjs 74 | hnZBn9DZmNFplm3R/JvLy44C2xald82XzjUP/YJ/9W+ufFNUuzDg+5BI0X6sQkSW++I+tC6mPkZR 75 | 5WW3cStXupDC5dTJmH50VVhO37sglYiEwm0uYu1NfRhmGWJXRRvmuO58UVfPY7afXk7R+mFrpx6I 76 | mcap6q+u7Ur/rfJFuRq6KcOsxz7T892ucbvMn1osd3VZnm9D+mEcq20Ssbumzl3bus3FQ7/Rafg/ 77 | Qi+/flpKoqJxMy5nFjPt0lx+zZDdRbfduuYqwLK0v256mc+TYv+x9ln5vW5uWSoidsiaKux33wkK 78 | l98hfD2uSxyPfwEAAP//AwBT7QgiQxIAAA== 79 | headers: 80 | Connection: 81 | - keep-alive 82 | Content-Encoding: 83 | - gzip 84 | Content-Type: 85 | - application/json 86 | Date: 87 | - Wed, 25 Mar 2020 14:24:21 GMT 88 | Server: 89 | - nginx 90 | Strict-Transport-Security: 91 | - max-age=31536000;includeSubDomains 92 | Transfer-Encoding: 93 | - chunked 94 | X-Content-Type-Options: 95 | - nosniff 96 | X-XSS-Protection: 97 | - 1;mode=block 98 | status: 99 | code: 200 100 | message: OK 101 | version: 1 102 | -------------------------------------------------------------------------------- /tests/cassettes/queryjob/test_poll_until_done_live_queryjob: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"queryString": "timechart()", "isLive": true}' 4 | headers: 5 | Accept: 6 | - '*/*' 7 | Accept-Encoding: 8 | - gzip, deflate 9 | Connection: 10 | - keep-alive 11 | Content-Length: 12 | - '46' 13 | Content-Type: 14 | - application/json 15 | User-Agent: 16 | - python-requests/2.22.0 17 | method: POST 18 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs 19 | response: 20 | body: 21 | string: !!binary | 22 | H4sIAAAAAAAAAKpWykxRslIyMtE1DM4MyMhNrTJ0M0orzDKtKk8sqvQ3Lg9T0lEqLE0FsvPCMlPL 23 | gWpLMnNTkzMSi0o0NJVqAQAAAP//AwAhg8h8QAAAAA== 24 | headers: 25 | Connection: 26 | - keep-alive 27 | Content-Encoding: 28 | - gzip 29 | Content-Type: 30 | - application/json 31 | Date: 32 | - Wed, 25 Mar 2020 14:24:21 GMT 33 | Server: 34 | - nginx 35 | Strict-Transport-Security: 36 | - max-age=31536000;includeSubDomains 37 | Transfer-Encoding: 38 | - chunked 39 | X-Content-Type-Options: 40 | - nosniff 41 | X-XSS-Protection: 42 | - 1;mode=block 43 | status: 44 | code: 200 45 | message: OK 46 | - request: 47 | body: null 48 | headers: 49 | Accept: 50 | - '*/*' 51 | Accept-Encoding: 52 | - gzip, deflate 53 | Connection: 54 | - keep-alive 55 | Content-Type: 56 | - application/json 57 | User-Agent: 58 | - python-requests/2.22.0 59 | method: GET 60 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs/24-1SiPhmez1F2fqj5zwaryO3wV 61 | response: 62 | body: 63 | string: !!binary | 64 | H4sIAAAAAAAAAIzYXW+bMBQG4L8y+XKiko9tsM1du/Ri0naxVdMupimixElRCXRgmnVV/vvMR9N2 65 | RePNVT6O7GM/GHLOI8uzKndl6TYs3WZl6yK2qSvHUt904b27d5VvWfrjka3zuqs8SxlnEVtfd/mt 66 | 6z9RbGIeW8n7FztG/w9MuAADicDAcUBgamHBEaUBA5UGA+MEDExiMFArMNCgMhaU0RyU0WPcsowm 67 | UEYLUEZLUEYrUEbHoIxOQBmtQRltUBkLyoy7CBxXw0EZQ6CMEaCMkaCMUaCMiUEZk4AyRoMyxqAy 68 | 434vnxljQRnLQRlLoIwVoIyVoIxVoIyNQRmbgDJ2ukstPrnsdGiAQFRmGnJpxHDpYDLECZMhLjAZ 69 | 4hKTIa4wGeIxJkM8wc4MTbuzeGZCICZD3GBnhrgFZcJqoOc1EYEyJEAZkqAMKVCGYlBmuhyXZSgB 70 | ZUiDMmRQGQvKCA7KCAJlhABlhARlhAJlpvO/LCNiUEYkoIzQoIwwqIwFZSQHZSSBMlKAMlKCMtMN 71 | d1lGKlBGxqCMTEAZqedkxJu6kGSoDfvX60cczUXaORr1dkjF52iSmUCaoxFvq9dw35uzmctyesgt 72 | 4yg5h2NmslQgjoqfcH5GbO98tsp8xtLHsUT/MNbmVoeS/bdvTj/6Yu/ym6zpK/W+og8JjJX7usxa 73 | /7qKp9MUp6D2LqvWN90+q4o/fYcgVPvv9kXVedc+jzQE7YuyLEKjgE3rjlhXpG2327nWu83Zodjs 74 | hnZBn9DZmNFplm3R/JvLy44C2xald82XzjUP/YJ/9W+ufFNUuzDg+5BI0X6sQkSW++I+tC6mPkZR 75 | 5WW3cStXupDC5dTJmH50VVhO37sglYgkPH9UxNqb+jDMMsSuijbMcd35oq6ex2w/vZyi9cPWTj0Q 76 | M41T1V9d25X+W+WLcjV0U4ZZj32m57td43aZP7VY7uqyPN+G9FkayluK2F1T565t3ebiod/nNPwd 77 | oeTF108rSULKw15czqxl2qS59JohuYtuu3XNVXBlaX/Z9DCfJ0SKw+faZ+X3urkNaUXskDVV2O++ 78 | ExQuv0P4elyXOB7/AgAA//8DAFpjEQRDEgAA 79 | headers: 80 | Connection: 81 | - keep-alive 82 | Content-Encoding: 83 | - gzip 84 | Content-Type: 85 | - application/json 86 | Date: 87 | - Wed, 25 Mar 2020 14:24:21 GMT 88 | Server: 89 | - nginx 90 | Strict-Transport-Security: 91 | - max-age=31536000;includeSubDomains 92 | Transfer-Encoding: 93 | - chunked 94 | X-Content-Type-Options: 95 | - nosniff 96 | X-XSS-Protection: 97 | - 1;mode=block 98 | status: 99 | code: 200 100 | message: OK 101 | - request: 102 | body: null 103 | headers: 104 | Accept: 105 | - '*/*' 106 | Accept-Encoding: 107 | - gzip, deflate 108 | Connection: 109 | - keep-alive 110 | Content-Length: 111 | - '0' 112 | Content-Type: 113 | - application/json 114 | User-Agent: 115 | - python-requests/2.22.0 116 | method: DELETE 117 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs/24-1SiPhmez1F2fqj5zwaryO3wV 118 | response: 119 | body: 120 | string: '' 121 | headers: 122 | Connection: 123 | - keep-alive 124 | Date: 125 | - Wed, 25 Mar 2020 14:24:22 GMT 126 | Server: 127 | - nginx 128 | Strict-Transport-Security: 129 | - max-age=31536000;includeSubDomains 130 | X-Content-Type-Options: 131 | - nosniff 132 | X-XSS-Protection: 133 | - 1;mode=block 134 | status: 135 | code: 204 136 | message: No Content 137 | version: 1 138 | -------------------------------------------------------------------------------- /tests/cassettes/queryjob/test_poll_until_done_live_queryjob_aggregate_query: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"queryString": "timechart()", "isLive": true}' 4 | headers: 5 | Accept: 6 | - '*/*' 7 | Accept-Encoding: 8 | - gzip, deflate 9 | Connection: 10 | - keep-alive 11 | Content-Length: 12 | - '46' 13 | Content-Type: 14 | - application/json 15 | User-Agent: 16 | - python-requests/2.22.0 17 | method: POST 18 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs 19 | response: 20 | body: 21 | string: !!binary | 22 | H4sIAAAAAAAAAKpWykxRslIyMtHN9qgyyYtK8XKNKg3N9QouLiw1DPAPLzJW0lEqLE0tqvTPC8tM 23 | LQeqLcnMTU3OSCwq0dBUqgUAAAD//wMAqnAbLEAAAAA= 24 | headers: 25 | Connection: 26 | - keep-alive 27 | Content-Encoding: 28 | - gzip 29 | Content-Type: 30 | - application/json 31 | Date: 32 | - Wed, 25 Mar 2020 14:37:37 GMT 33 | Server: 34 | - nginx 35 | Strict-Transport-Security: 36 | - max-age=31536000;includeSubDomains 37 | Transfer-Encoding: 38 | - chunked 39 | X-Content-Type-Options: 40 | - nosniff 41 | X-XSS-Protection: 42 | - 1;mode=block 43 | status: 44 | code: 200 45 | message: OK 46 | - request: 47 | body: null 48 | headers: 49 | Accept: 50 | - '*/*' 51 | Accept-Encoding: 52 | - gzip, deflate 53 | Connection: 54 | - keep-alive 55 | Content-Type: 56 | - application/json 57 | User-Agent: 58 | - python-requests/2.22.0 59 | method: GET 60 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs/24-kHz4nZdJEZuUmJSsqu1POWr3 61 | response: 62 | body: 63 | string: !!binary | 64 | H4sIAAAAAAAAAIzYz2+bMBQH8H9l8nGikp9tsM2tXXqYtB22atphmiJKnNQqgQ5Ms63q/z4TaNqu 65 | SHxzyo8n+9kfDHnvgZVFXbqqchuWb4uqcwnbNLVjeWj7+N7duzp0LP/xwNZl09eB5YyzhK2v+/LW 66 | DZ8oNSnPuODDiz0mC4FEYOA4IDCisOCI0oCBSoOBaQYGZikYqBUYaCQYaEEZzUEZPcYty2gCZbQA 67 | ZbQEZbQCZXQKyugMlNEalNEGlbGgzLiLwCk0HJQxBMoYAcoYCcoYBcqYFJQxGShjNChjDCoz7vfy 68 | mTEWlLEclLEEylgBylgJylgFytgUlLEZKGOnu9TiA8lOhwYIRGWmIZdGjJcOJkOcMBniApMhLjEZ 69 | 4gqTIZ5iMsQz7MzQtDuLZyYGYjLEDXZmiFtQJq4GegwTEShDApQhCcqQAmUoBWWmy3FZhjJQhjQo 70 | QwaVsaCM4KCMIFBGCFBGSFBGKFBmOv/LMiIFZUQGyggNygiDylhQRnJQRhIoIwUoIyUoM91wl2Wk 71 | AmVkCsrIDJSRek5GvCn3SMaSb3i9fsTRXKSdo1Fvh1R8jiabCaQ5GvG2KI33vTmbuSynh9wyjpJz 72 | OGYmSwXiqBTEUdkTzs+E7V0oVkUoWP4wlugfxtrc6liy/w7t6cfg9668KdqhUh8q+pjpWLmvq6IL 73 | r6t4Ok1xCuruinp90++L2v8dOgSx2n+393UfXPc80jFo76vKx0YBmzYoYb3Pu363c11wm7OD3+yO 74 | 7YIhobMxo9MsW9/+n8vLjgLb+iq49kvv2j/Dgn8Nb65C6+tdHPB9TMR3H+sYUZTB38fWxdTH8HVZ 75 | 9Ru3cpWLKVxOnYzpR1fH5Qy9C4rVfqqtFgnrbprDcZZj7Mp3cY7rPvimfh6z+/Ryii4ct3bqgWTT 76 | OHXz1XV9Fb7VwVerYzflOOvjkOn5bte6XRFOLZa7pqrOtzF9lguTZgm7a5vSdZ3bXPwZ9jmP/1vo 77 | 5ddPK8lUMu7F5cxapk2aS689JnfRb7euvYquLB8umwHm84QoefzchKL63rS3Ma2EHYq2jvs9dILi 78 | 5XeIX4/rEo+P/wAAAP//AwCza1CuQxIAAA== 79 | headers: 80 | Connection: 81 | - keep-alive 82 | Content-Encoding: 83 | - gzip 84 | Content-Type: 85 | - application/json 86 | Date: 87 | - Wed, 25 Mar 2020 14:37:38 GMT 88 | Server: 89 | - nginx 90 | Strict-Transport-Security: 91 | - max-age=31536000;includeSubDomains 92 | Transfer-Encoding: 93 | - chunked 94 | X-Content-Type-Options: 95 | - nosniff 96 | X-XSS-Protection: 97 | - 1;mode=block 98 | status: 99 | code: 200 100 | message: OK 101 | - request: 102 | body: null 103 | headers: 104 | Accept: 105 | - '*/*' 106 | Accept-Encoding: 107 | - gzip, deflate 108 | Connection: 109 | - keep-alive 110 | Content-Length: 111 | - '0' 112 | Content-Type: 113 | - application/json 114 | User-Agent: 115 | - python-requests/2.22.0 116 | method: DELETE 117 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs/24-kHz4nZdJEZuUmJSsqu1POWr3 118 | response: 119 | body: 120 | string: '' 121 | headers: 122 | Connection: 123 | - keep-alive 124 | Date: 125 | - Wed, 25 Mar 2020 14:37:38 GMT 126 | Server: 127 | - nginx 128 | Strict-Transport-Security: 129 | - max-age=31536000;includeSubDomains 130 | X-Content-Type-Options: 131 | - nosniff 132 | X-XSS-Protection: 133 | - 1;mode=block 134 | status: 135 | code: 204 136 | message: No Content 137 | version: 1 138 | -------------------------------------------------------------------------------- /tests/cassettes/queryjob/test_poll_until_done_static_queryjob_non_aggregate_query: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"queryString": "", "isLive": false}' 4 | headers: 5 | Accept: 6 | - '*/*' 7 | Accept-Encoding: 8 | - gzip, deflate 9 | Connection: 10 | - keep-alive 11 | Content-Length: 12 | - '36' 13 | Content-Type: 14 | - application/json 15 | User-Agent: 16 | - python-requests/2.22.0 17 | method: POST 18 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs 19 | response: 20 | body: 21 | string: !!binary | 22 | H4sIAAAAAAAAAKpWykxRslIyMtHN9MozDagoiijNDPTO986OcMxPS7RIdq9U0lEqLE0tqvTPC8tM 23 | LQeq1VKqBQAAAP//AwDsoBFLNgAAAA== 24 | headers: 25 | Connection: 26 | - keep-alive 27 | Content-Encoding: 28 | - gzip 29 | Content-Type: 30 | - application/json 31 | Date: 32 | - Wed, 25 Mar 2020 14:33:12 GMT 33 | Server: 34 | - nginx 35 | Strict-Transport-Security: 36 | - max-age=31536000;includeSubDomains 37 | Transfer-Encoding: 38 | - chunked 39 | X-Content-Type-Options: 40 | - nosniff 41 | X-XSS-Protection: 42 | - 1;mode=block 43 | status: 44 | code: 200 45 | message: OK 46 | - request: 47 | body: null 48 | headers: 49 | Accept: 50 | - '*/*' 51 | Accept-Encoding: 52 | - gzip, deflate 53 | Connection: 54 | - keep-alive 55 | Content-Type: 56 | - application/json 57 | User-Agent: 58 | - python-requests/2.22.0 59 | method: GET 60 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs/24-iJn5PxrXuiQKoKkXAofa8cGy 61 | response: 62 | body: 63 | string: !!binary | 64 | H4sIAAAAAAAAAOycXVPjNhSG/0rGvWsh0Ydly74qX9OG7nZZPnbZLjuMSUTwYuxgOxBg+O89DpGc 65 | mSJDLyKiwVzZUSLbD6+Pzitb58EZROlAJIkYOuF5lBRizRlmqXDCMp/AtrgRaVk44fcH5/cyvhJF 66 | GV2NnRAzzrDLmBfQwF1zfinvxvAT5/LGgZ1cjDPYKaJ0eJZNTyfx/rY4PvKPvpV70/v+7u7tdLp5 67 | vfkBvjrr8n52OOefaj+Pbosyj9MRfIAD0sUe7+Jul+DOemdSiJx0viPS+zu76RGE/RDT0A1CFHR+ 68 | Q/D3o3Pi7H06OOz0LiZXcdaLxnHvBvegNzjtnkiioowH62eT5LLz5+HhXg938YnTIQh1EPxyHbZP 69 | nOog0QiuGfZQF2HaYR572qrOLwZKzgH99sdG/+doM5uOjne3+le8/yXoD8gpOvVPazLO49pbQnOD 70 | OTP8X2Y8pHR5zNyOx/wZM/c1zLxXMeO+1wqtFhqroXG90IxAs0Zo7iuYuZxyRNuIpiIaldCAjE5o 71 | 0ORzf+nQrBEaeRUzwglaOjN7hk5cQyN6oRGOW6GpmxO9zIwS5BFK3p3QBj/HO9dfP3/Z/+vs04CM 72 | 2f7xWT7cSk7xKZVJWoVGozRT1FYspDVAk1laMzTmu+9v8GygJtM0oMb0UjNCzR6pyTytCRpiAadB 73 | G9XqqCYTNQpodFIzRM0eqclMrRka9xlvpVZLTaZqQE1nPqsmE9TskZrM1RqhUeZ7788U6AdQEswz 74 | XIqodgCFJhPUrJEa4a+Dhjluo5qKakTZgmapmaBmj9SULdBDI0EAc7etLagdKJG2oEKjydVMUbNH 75 | atIWvAANI9ZGtTqqSVuwAtTskZq0BU3QOOeYtrnaQlSTtoAAGl1UM0TNHqlJW9AMzffdNqrVUsPS 76 | FgA17QAKTSaoWSM1LG1BEzQWeIHXPi1YkJq0BQTQ6KKaIWr2SE3agheguaidV1uQmrIFzVIzQc0e 77 | qSlb0ASNeT5b/nPjVXvYrp9Xw8oWwDt82qhmhpo9UlO2oBEa84N2smMhqilbwJhuCpdAkwlq9khN 78 | 2YIGaNTlvtsOoLXUlCsAMrqgZgiaNUpTpqCJGfUIaU1BLTTlCSjVjp7QZAKaNUJTlqCBGfIpe4cv 79 | RerzNOUIgIwuohmCZo3QlCFoYsZ85LVDZx3RlB9ATDuhBk0moFkjNGUH9Mywx1jgtk8JFuzA/N2E 80 | iowmopmCZo3QpBloYIY8PyB+O8UhhQbIZJJWodEozRQ1S5QG0GSW1giNuZy1MW1BajJNQx5ztVIz 81 | Q80eqck8rQkarJFyXW/p73MwV79WNggxWt5aWQZrZWH9MayVZfVaWZ0fgPtT5mkIyOiUZgiaLbPd 82 | QE1maitAzZ77U87brgA0tf7/2bXsxFvq/em5/+P+lKlaIzTqErz8tSvWBDU5bYsI1Y6e0GSCmT0x 83 | Tc7brgA0a0KacgRvLzRrIpoyBE3MoOgJWv5aAmsCmrIDBLK656vAIGgywcyegKbswNtDsyag1W7g 84 | zYVmTUCrvYCOGQ0QgqpQ7UscCzMcT7O2MzLPR7Q5tOWPAtbcnHMrMGf2Y825EmW0HZWREz481VXb 85 | yiZp6YRVYSsxLXPVeBEVH7Nc7MxrrzmzumyA3TmPk1Lknyciv6s6ua42DmS9tF9hAiEu+il8IxqU 86 | 8Q0UVZsXdIvTQTIZim2RiFIMZbfzRpFCTbOnqmXVHDKv1jQUF9nt7Ciz727HVU22s0kZZ2ndZ/Fh 87 | 8RBQFy6HS6n6QR6i837SbF8Uk6Q8Sss42Z7VeZsdFS4lLjZGo1yMorI+z3GWJBvncP7QUeBTmPcZ 88 | 59lAFIUYbt6VAsrQYcTx4sfyWiqEMxo7z1zNHNNzJ5jPTm9zcn4u8oP4Hk6l6qmqSvcxTpIYjggP 89 | CcusjJKvWX7phPASxG2Up1BSrqqJB//TW/j46cLI4+O/AAAA//8DAAz4EphNTwAA 90 | headers: 91 | Connection: 92 | - keep-alive 93 | Content-Encoding: 94 | - gzip 95 | Content-Type: 96 | - application/json 97 | Date: 98 | - Wed, 25 Mar 2020 14:33:13 GMT 99 | Server: 100 | - nginx 101 | Strict-Transport-Security: 102 | - max-age=31536000;includeSubDomains 103 | Transfer-Encoding: 104 | - chunked 105 | X-Content-Type-Options: 106 | - nosniff 107 | X-XSS-Protection: 108 | - 1;mode=block 109 | status: 110 | code: 200 111 | message: OK 112 | version: 1 113 | -------------------------------------------------------------------------------- /src/humiolib/cli.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module that contains the command line app. 3 | 4 | Why does this file exist, and why not put this in __main__? 5 | 6 | You might be tempted to import things from __main__ later, but that will cause 7 | problems: the code will get executed twice: 8 | 9 | - When you run `python -mhumiolib` python will execute 10 | ``__main__.py`` as a script. That means there won't be any 11 | ``humiolib.__main__`` in ``sys.modules``. 12 | - When you import __main__ it will get executed again (as a module) because 13 | there's no ``humiolib.__main__`` in ``sys.modules``. 14 | 15 | Also see (1) from http://click.pocoo.org/5/setuptools/#setuptools-integration 16 | """ 17 | import argparse 18 | import json 19 | from humiolib.HumioClient import HumioClient, HumioIngestClient 20 | from humiolib.HumioExceptions import HumioException 21 | 22 | def command_ingest(client, args): 23 | api = client 24 | 25 | fields = dict(args.field) if args.field else None 26 | tags = dict(args.tag) if args.tag else None 27 | 28 | if fields: 29 | print("[*] Adding fields: ") 30 | print(json.dumps(fields, indent=2)) 31 | 32 | if tags: 33 | print("[*] Adding tags: ") 34 | print(json.dumps(tags, indent=2)) 35 | 36 | if args.parser: 37 | print("[*] Applying parser '" + args.parser + "'") 38 | 39 | if args.file: 40 | print("[*] Uploading events from '" + args.file + "'") 41 | with open(args.file) as sourceFile: 42 | events = list( 43 | line.strip() for line in sourceFile.readlines() if line.strip() 44 | ) 45 | try: 46 | api.ingest_messages( 47 | messages=events, parser=args.parser, fields=fields, tags=tags 48 | ) 49 | print("[OK]") 50 | except HumioException as e: 51 | print(e) 52 | 53 | if args.interactive: 54 | print("[*] Starting interactive mode") 55 | inp = input(">>") 56 | while inp: 57 | try: 58 | api.ingest_messages( 59 | messages=[inp], parser=args.parser, fields=fields, tags=tags 60 | ) 61 | print("[OK]") 62 | except HumioException as e: 63 | print(e) 64 | inp = input(">>") 65 | 66 | 67 | parser = argparse.ArgumentParser(description="Command description.") 68 | parser.add_argument( 69 | "--host", 70 | dest="host", 71 | default="https://cloud.humio.com", 72 | help="Humio host (default: https://cloud.humio.com)", 73 | ) 74 | parser.add_argument( 75 | "-r", "--repository", default=None, help="virtual or real repository name" 76 | ) 77 | 78 | token_group = parser.add_mutually_exclusive_group() 79 | token_group.add_argument("-u", "--user-token", default=None, help="Humio user token") 80 | token_group.add_argument("-t", "--ingest-token", default=None, help="Humio ingest token") 81 | 82 | commands = parser.add_subparsers(help="Possible commands") 83 | ingest_parser = commands.add_parser("ingest", help="Ingest data to Humio") 84 | ingest_parser.set_defaults(func=command_ingest) 85 | 86 | 87 | ingest_parser.add_argument( 88 | "-p", 89 | "--parser", 90 | default=None, 91 | help="The parser to use (this does not work if there is a parser attached to the ingest token)", 92 | ) 93 | ingest_parser.add_argument( 94 | "--field", 95 | default=[], 96 | action="append", 97 | nargs=2, 98 | metavar=("FIELD", "VALUE"), 99 | help="Add a field to all events (example: `--field source cli`)", 100 | ) 101 | ingest_parser.add_argument( 102 | "--tag", 103 | default=[], 104 | action="append", 105 | nargs=2, 106 | metavar=("TAG", "VALUE"), 107 | help="Add a tag to all events (example: `--tag source cli`) NOTE: This may have performance impacts on Humio", 108 | ) 109 | 110 | ingest_parser.add_argument( 111 | "-f", "--file", default=None, help="Path to file with raw events on each line" 112 | ) 113 | ingest_parser.add_argument( 114 | "-i", 115 | "--interactive", 116 | default=False, 117 | action="store_true", 118 | help="Start REPL to send one event per line", 119 | ) 120 | 121 | 122 | def main(args=None): 123 | args = parser.parse_args(args=args) 124 | 125 | if args.ingest_token: 126 | client = HumioIngestClient( 127 | base_url=args.host, 128 | ingest_token=args.ingest_token 129 | ) 130 | else: 131 | client = HumioClient( 132 | base_url=args.host, 133 | repository=args.repository, 134 | user_token=args.user_token 135 | ) 136 | 137 | args.func(client, args) 138 | -------------------------------------------------------------------------------- /tests/test_humioclient.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import vcr 3 | import os 4 | from humiolib import HumioClient, HumioIngestClient 5 | from humiolib.HumioExceptions import HumioHTTPException 6 | 7 | user_token = os.environ['HUMIO_USER_TOKEN'] if 'HUMIO_USER_TOKEN' in os.environ else "bogustoken" 8 | ingest_token = os.environ['HUMIO_INGEST_TOKEN'] if 'HUMIO_INGEST_TOKEN' in os.environ else "bogustoken" 9 | 10 | dirname = os.path.dirname(__file__) 11 | cassettedir = os.path.join(dirname, 'cassettes/humioclient') 12 | 13 | # HUMIOCLIENT TESTS 14 | @pytest.fixture 15 | def humioclient(): 16 | client = HumioClient( 17 | base_url= "https://cloud.humio.com", 18 | repository= "sandbox", 19 | user_token=user_token, 20 | ) 21 | return client 22 | 23 | 24 | @pytest.fixture 25 | def ingestclient(): 26 | client = HumioIngestClient( 27 | base_url= "https://cloud.humio.com", 28 | ingest_token=ingest_token, 29 | ) 30 | return client 31 | 32 | @vcr.use_cassette(cassette_library_dir=cassettedir, filter_headers=['Authorization']) 33 | def test_create_queryjob_success(humioclient): 34 | queryjob = humioclient.create_queryjob("timechart()") 35 | assert queryjob 36 | 37 | @vcr.use_cassette(cassette_library_dir=cassettedir, filter_headers=['Authorization']) 38 | def test_create_queryjob_incorrect_query_syntax(humioclient): 39 | with pytest.raises(HumioHTTPException): 40 | humioclient.create_queryjob("timechart(func=nowork)") 41 | 42 | @vcr.use_cassette(cassette_library_dir=cassettedir, filter_headers=['Authorization']) 43 | def test_streaming_query_success(humioclient): 44 | result = [] 45 | for entry in humioclient.streaming_query("timechart()"): 46 | result.append(entry) 47 | assert len(result) != 0 48 | 49 | 50 | @vcr.use_cassette(cassette_library_dir=cassettedir, filter_headers=['Authorization']) 51 | def test_ingest_messages_on_humioclient_success(humioclient): 52 | messages = [ 53 | "192.168.1.49 - user1 [02/Nov/2017:13:48:33 +0000] \"POST /humio/api/v1/ingest/elastic-bulk HTTP/1.1\" 200 0 \"-\" \"useragent\" 0.014 657 0.014", 54 | "192.168.1..21 - user2 [02/Nov/2017:13:49:09 +0000] \"POST /humio/api/v1/ingest/elastic-bulk HTTP/1.1\" 200 0 \"-\" \"useragent\" 0.013 565 0.013", 55 | ] 56 | 57 | response = humioclient.ingest_messages(messages) 58 | assert response == {} 59 | 60 | @vcr.use_cassette(cassette_library_dir=cassettedir, filter_headers=['Authorization']) 61 | def test_ingest_json_on_humioclient_success(humioclient): 62 | data = [ 63 | { 64 | "tags": { 65 | "host": "server1", 66 | "source": "application.log", 67 | }, 68 | "events": [ 69 | { 70 | "timestamp": "2020-03-23T00:00:00+00:00", 71 | "attributes": { 72 | "key1": "value1", 73 | "key2": "value2" 74 | } 75 | } 76 | ] 77 | } 78 | ] 79 | 80 | response = humioclient.ingest_json_data(data) 81 | assert response == {} 82 | 83 | @vcr.use_cassette(cassette_library_dir=cassettedir, filter_headers=['Authorization']) 84 | def test_get_status(humioclient): 85 | response = humioclient.get_status() 86 | assert list(response.keys()) == ["status", "version"] 87 | 88 | 89 | # INGEST CLIENT TESTS 90 | @vcr.use_cassette(cassette_library_dir=cassettedir, filter_headers=['Authorization']) 91 | def test_ingest_messages_on_ingestclient_ssuccess(ingestclient): 92 | messages = [ 93 | "192.168.1.49 - user1 [02/Nov/2017:13:48:33 +0000] \"POST /humio/api/v1/ingest/elastic-bulk HTTP/1.1\" 200 0 \"-\" \"useragent\" 0.014 657 0.014", 94 | "192.168.1..21 - user2 [02/Nov/2017:13:49:09 +0000] \"POST /humio/api/v1/ingest/elastic-bulk HTTP/1.1\" 200 0 \"-\" \"useragent\" 0.013 565 0.013", 95 | ] 96 | 97 | response = ingestclient.ingest_messages(messages) 98 | assert response == {} 99 | 100 | @vcr.use_cassette(cassette_library_dir=cassettedir, filter_headers=['Authorization']) 101 | def test_ingest_json_on_ingestclient_success(ingestclient): 102 | data = [ 103 | { 104 | "tags": { 105 | "host": "server1", 106 | "source": "application.log", 107 | }, 108 | "events": [ 109 | { 110 | "timestamp": "2020-03-23T00:00:00+00:00", 111 | "attributes": { 112 | "key1": "value1", 113 | "key2": "value2" 114 | } 115 | } 116 | ] 117 | } 118 | ] 119 | 120 | response = ingestclient.ingest_json_data(data) 121 | assert response == {} 122 | 123 | 124 | -------------------------------------------------------------------------------- /src/humiolib/WebCaller.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import functools 3 | from humiolib.HumioExceptions import HumioConnectionException, HumioHTTPException, HumioTimeoutException, HumioConnectionDroppedException 4 | 5 | HTTPError = requests.exceptions.HTTPError 6 | ConnectionError = requests.exceptions.ConnectionError 7 | TimeoutError = requests.exceptions.Timeout 8 | ChunkingError = requests.exceptions.ChunkedEncodingError 9 | 10 | class WebCaller: 11 | """ 12 | Object used for abstracting calls to the Humio API 13 | """ 14 | version_number_humio = "v1" 15 | 16 | def __init__(self, base_url): 17 | """ 18 | :param base_url: URL of Humio instance. 19 | :type func: string 20 | """ 21 | self.base_url = base_url 22 | self.rest_url = "{}/api/{}/".format(self.base_url, self.version_number_humio) 23 | self.graphql_url = "{}/graphql".format(self.base_url) 24 | 25 | 26 | def call_rest(self, verb, endpoint, headers=None, data=None, files=None, stream=False, **kwargs): 27 | """ 28 | Call one of Humio's REST endpoints 29 | 30 | :param verb: Http verb 31 | :type verb: str 32 | :param endpoint: Called Humio endpoint 33 | :type endpoint: str 34 | :param headers: Http headers 35 | :type headers: dict, optional 36 | :param data: Post request body 37 | :type data: dict, optional 38 | :param files: Files to be posted 39 | :type files: dict, optional 40 | :param stream: Indicates whether a stream request should be made 41 | :type stream: bool, optional 42 | 43 | :return: Response to web request 44 | :rtype: Response Object 45 | """ 46 | link = self.rest_url + endpoint 47 | return self._make_request(verb, link, headers, data, files, stream, **kwargs) 48 | 49 | def call_graphql(self, headers=None, data=None, **kwargs): 50 | """ 51 | Call Humio's GraphQL endpoint 52 | 53 | :param headers: Http headers 54 | :type headers: dict, optional 55 | :param data: Post request body for GraphQL 56 | :type data: dict, optional 57 | 58 | :return: Response to web request 59 | :rtype: Response Object 60 | """ 61 | return self._make_request("post", self.graphql_url, headers, data, **kwargs) 62 | 63 | def _make_request(self, verb, link, headers=None, data=None, files=None, stream=False, **kwargs): 64 | """ 65 | Make a webrequest. 66 | By creating custom errors here, we ensure that calling code will not have to depend 67 | on the types of errors thrown by the httplibrary. Thus allowing us to more easily, 68 | switch out the library in the future. 69 | 70 | :param verb: Http verb 71 | :type verb: str 72 | :param endpoint: Called Humio endpoint 73 | :type endpoint: str 74 | :param headers: Http headers 75 | :type headers: dict, optional 76 | :param data: Post request body 77 | :type data: dict, optional 78 | :param files: Files to be posted 79 | :type files: dict, optional 80 | :param stream: Indicates whether a stream request should be made 81 | :type stream: bool, optional 82 | 83 | :return: Response to web request 84 | :rtype: Response Object 85 | """ 86 | try: 87 | response = requests.request( 88 | verb, link, data=data, headers=headers, stream=stream, files=files, **kwargs 89 | ) 90 | response.raise_for_status() 91 | except ConnectionError as e: 92 | raise HumioConnectionException(e) 93 | except HTTPError as e: 94 | raise HumioHTTPException(e.response.text, e.response.status_code) 95 | except TimeoutError as e: 96 | raise HumioTimeoutException(e) 97 | 98 | return response 99 | 100 | @staticmethod 101 | def response_as_json(func): 102 | """ 103 | Wrapper to take the raw requests responses and turn them into json 104 | 105 | :param func: Function to be wrapped. 106 | :type func: Function 107 | 108 | :return: Result of function, parsed into python objects from json 109 | :rtype: dict 110 | """ 111 | @functools.wraps(func) 112 | def wrapper(*args, **kwargs): 113 | resp = func(*args, **kwargs) 114 | return resp.json() 115 | 116 | return wrapper 117 | 118 | 119 | class WebStreamer(): 120 | """ 121 | Wrapper for a web request stream. 122 | Its main purpose is to catch errors during stream and raise them again as custom Humio exceptions. 123 | """ 124 | def __init__(self, connection): 125 | """ 126 | :param connection: Connection object created by http library. 127 | :type connection: Connection 128 | """ 129 | self.connection = connection.iter_lines() 130 | 131 | def __iter__(self): 132 | return self 133 | 134 | def __next__(self): 135 | try: 136 | return next(self.connection) 137 | # This error occurs during live queries, when data hasn't been streamed in a while 138 | except ChunkingError: 139 | raise HumioConnectionDroppedException("Connection to streaming socket was lost") -------------------------------------------------------------------------------- /tests/cassettes/queryjob/test_poll_until_done_static_queryjob: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"queryString": "timechart()", "isLive": false}' 4 | headers: 5 | Accept: 6 | - '*/*' 7 | Accept-Encoding: 8 | - gzip, deflate 9 | Connection: 10 | - keep-alive 11 | Content-Length: 12 | - '47' 13 | Content-Type: 14 | - application/json 15 | User-Agent: 16 | - python-requests/2.22.0 17 | method: POST 18 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs 19 | response: 20 | body: 21 | string: !!binary | 22 | H4sIAAAAAAAAAKpWykxRslIyMtFN8UxMd/YNzM6tMPAoLwv2cDcpN0kqCXJR0lEqLE0tqvTPC8tM 23 | LQeqLcnMTU3OSCwq0dBUqgUAAAD//wMADKcwHkAAAAA= 24 | headers: 25 | Connection: 26 | - keep-alive 27 | Content-Encoding: 28 | - gzip 29 | Content-Type: 30 | - application/json 31 | Date: 32 | - Wed, 25 Mar 2020 14:24:17 GMT 33 | Server: 34 | - nginx 35 | Strict-Transport-Security: 36 | - max-age=31536000;includeSubDomains 37 | Transfer-Encoding: 38 | - chunked 39 | X-Content-Type-Options: 40 | - nosniff 41 | X-XSS-Protection: 42 | - 1;mode=block 43 | status: 44 | code: 200 45 | message: OK 46 | - request: 47 | body: null 48 | headers: 49 | Accept: 50 | - '*/*' 51 | Accept-Encoding: 52 | - gzip, deflate 53 | Connection: 54 | - keep-alive 55 | Content-Type: 56 | - application/json 57 | User-Agent: 58 | - python-requests/2.22.0 59 | method: GET 60 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs/24-dIagCMQkmx0HwvSHG4w4btRD 61 | response: 62 | body: 63 | string: !!binary | 64 | H4sIAAAAAAAAAGySQU/DMAyF/wryEXVSGZSN3hjjgAQHQIgDQlNo3c5amozEYYxp/x2nLR1C9JQ4 65 | lv2997qDQpkCtcYS8kppjwmU1uBwwQ807CF/eU2gQVZzxQryHbT1KxsMQ54mgJ/shjemBoulcvIE 66 | 7AJCAm+hWCEvtPK86M7ydpJNs5OzbJLG79Dk18oslqFRhr4ilvQdNWQCo//T1JDWJHBw8TMiUO5D 67 | XaNnLEcbKmtsIQRo1BENKBW5vyxpdnHaDdonUJFmdPcB3TbqfY+HR3Zkahl4LCDkb4x0qILp4+AX 68 | mUKHEueoURCue/d6Z9GInE71+TibTM+yBPzSbtotbe+cvOx4C0zWDBmQv/29wnNrbZwjxNN+jrEP 69 | 6IPmJ8Ok54cIRQr5y7p2WCsWzhhIAmur9WUl+JCPp+lYCs4W6D2Ws230OWY6lH5USK214fofGb0/ 70 | /5G5lmsWqgrdo0TaDo+R3PXxTeRqWeln61bCk8BGOSNG97/dRsqdoHS//wYAAP//AwBnbBs2sQIA 71 | AA== 72 | headers: 73 | Connection: 74 | - keep-alive 75 | Content-Encoding: 76 | - gzip 77 | Content-Type: 78 | - application/json 79 | Date: 80 | - Wed, 25 Mar 2020 14:24:18 GMT 81 | Server: 82 | - nginx 83 | Strict-Transport-Security: 84 | - max-age=31536000;includeSubDomains 85 | Transfer-Encoding: 86 | - chunked 87 | X-Content-Type-Options: 88 | - nosniff 89 | X-XSS-Protection: 90 | - 1;mode=block 91 | status: 92 | code: 200 93 | message: OK 94 | - request: 95 | body: null 96 | headers: 97 | Accept: 98 | - '*/*' 99 | Accept-Encoding: 100 | - gzip, deflate 101 | Connection: 102 | - keep-alive 103 | Content-Type: 104 | - application/json 105 | User-Agent: 106 | - python-requests/2.22.0 107 | method: GET 108 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs/24-dIagCMQkmx0HwvSHG4w4btRD 109 | response: 110 | body: 111 | string: !!binary | 112 | H4sIAAAAAAAAAIzYz2+bMBQH8H9l8nGikp8N2ObWLj1M2g5bNe0wTRElTmqVQAemWVflf5/50bRd 113 | 0fjmlJIn+9kfDH3vkRV5VdiytBuWbfOytRHb1JVlmW+68N3e28q3LPvxyNZF3VWeZYyziK2vu+LW 114 | 9n9RohOeGMn7DztG/w9MuQADicDAcUBgamHAEaUGA2MFBiYpGJgmYKCKwUCNyhhQRnFQRo1xyzKK 115 | QBklQBklQRkVgzIqAWVUCsooBcoojcoYUGbcReC4ag7KaAJltABltARldAzK6ASU0SkooxUoozUq 116 | M+738pnRBpQxHJQxBMoYAcoYCcqYGJQxCShjUlDGTE+pxTeXmQ4NEIjKTEMujRhuHUyGOGEyxAUm 117 | Q1xiMsRjTIZ4gskQT7EzQ9PuLJ6ZEIjJENfYmSFuQJmwGuh9TUSgDAlQhiQoQzEoQwkoM92OyzKU 118 | gjKkQBnSqIwBZQQHZQSBMkKAMkKCMiIGZabzvywjElBGpKCMUKCM0KiMAWUkB2UkgTJSgDJSgjLT 119 | A3dZRsagjExAGZmCMlLNyYg3dSHJUBv2n9evOJqLNHM08dshYz5Hk84E0hyNeFu9hufenM1cltNL 120 | bhknlnM4eibLGMSJkyecnxHbW5+vcp+z7HEs0T+MtblRoWT/7ZvTj97tbXGTN32l3lf0IYGxcl+X 121 | eetfV/F0muIU1N7l1fqm2+eV+9N3CEK1/27vqs7b9nmkIWjvytKFRgGb1h2xzmVtt9vZ1tvN2cFt 122 | dkO7oE/obMzoNMvWNf/m8rKjwLau9Lb50tnmoV/wr/7LlW9ctQsDvg+JuPZjFSLywrv70LqY+hiu 123 | KspuY1e2tCGFy6mTMf1oq7CcvndBcSoSFYqIiLU39WGYZYhduTbMcd15V1fPY7afXk7R+mFrpx6I 124 | nsap6q+27Ur/rfKuXA3dlGHWY5/p+W7X2F3uTy2Wu7osz7ch/TCOSSnkcdfUhW1bu7l46Dc6C/+P 125 | UPri8tNS0jgaN+NyZjHTLs3l1wzZXXTbrW2uAizL+vuml/k8KYZqzoQrtc/L73VzyzIRsUPeVGHL 126 | +2ZQuAMP4fK4NHE8/gUAAP//AwCnPgnJRhIAAA== 127 | headers: 128 | Connection: 129 | - keep-alive 130 | Content-Encoding: 131 | - gzip 132 | Content-Type: 133 | - application/json 134 | Date: 135 | - Wed, 25 Mar 2020 14:24:21 GMT 136 | Server: 137 | - nginx 138 | Strict-Transport-Security: 139 | - max-age=31536000;includeSubDomains 140 | Transfer-Encoding: 141 | - chunked 142 | X-Content-Type-Options: 143 | - nosniff 144 | X-XSS-Protection: 145 | - 1;mode=block 146 | status: 147 | code: 200 148 | message: OK 149 | version: 1 150 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ====================== 2 | Humiolib 3 | ====================== 4 | 5 | .. start-badges 6 | 7 | |docs| |version| |license| 8 | 9 | 10 | .. |docs| image:: https://readthedocs.org/projects/python-humio/badge/?style=flat 11 | :target: https://readthedocs.org/projects/python-humio 12 | :alt: Documentation Status 13 | 14 | .. |version| image:: https://img.shields.io/pypi/v/humiolib.svg 15 | :target: https://pypi.org/project/humiolib 16 | :alt: PyPI Package latest release 17 | 18 | .. |license| image:: https://img.shields.io/badge/License-Apache%202.0-blue.svg 19 | :target: https://opensource.org/licenses/Apache-2.0 20 | :alt: Apache 2.0 License 21 | 22 | .. end-badges 23 | 24 | The `humiolib` library is a wrapper for Humio's web API, supporting easy interaction with Humio directly from Python. 25 | Full documentation for this repository can be found at https://python-humio.readthedocs.io/en/latest/readme.html. 26 | 27 | Vision 28 | ====== 29 | The vision for `humiolib` is to create an opinionated wrapper around the Humio web API, supporting log ingestion and log queries. 30 | The project does not simply expose web endpoints as Python methods, but attempts to improve upon the usability experience of the API. 31 | In addition the project seeks to add non-intrusive quality of life features, so that users can focus on their primary goals during development. 32 | 33 | Governance 34 | ========== 35 | This project is maintained by employees at Humio ApS. 36 | As a general rule, only employees at Humio can become maintainers and have commit privileges to this repository. 37 | Therefore, if you want to contribute to the project, which we very much encourage, you must first fork the repository. 38 | Maintainers will have the final say on accepting or rejecting pull requests. 39 | As a rule of thumb, pull requests will be accepted if: 40 | 41 | * The contribution fits with the project's vision 42 | * All automated tests have passed 43 | * The contribution is of a quality comparable to the rest of the project 44 | 45 | The maintainers will attempt to react to issues and pull requests quickly, but their ability to do so can vary. 46 | If you haven't heard back from a maintainer within 7 days of creating an issue or making a pull request, please feel free to ping them on the relevant post. 47 | 48 | The active maintainers involved with this project include: 49 | 50 | * `Alexander Brandborg `_ 51 | 52 | Installation 53 | ============ 54 | The `humiolib` library has been published on PyPI, so you can use `pip` to install it: 55 | :: 56 | 57 | pip install humiolib 58 | 59 | 60 | Usage 61 | ======== 62 | The examples below seek to get you going with `humiolib`. 63 | For further documentation have a look at the code itself. 64 | 65 | HumioClient 66 | *********** 67 | The HumioClient class is used for general interaction with Humio. 68 | It is mainly used for performing queries, as well as managing different aspects of your Humio instance. 69 | 70 | .. code-block:: python 71 | 72 | from humiolib.HumioClient import HumioClient 73 | 74 | # Creating the client 75 | client = HumioClient( 76 | base_url= "https://cloud.humio.com", 77 | repository= "sandbox", 78 | user_token="*****") 79 | 80 | # Using a streaming query 81 | webStream = client.streaming_query("Login Attempt Failed", is_live=True) 82 | for event in webStream: 83 | print(event) 84 | 85 | # Using a queryjob 86 | queryjob = client.create_queryjob("Login Attempt Failed", is_live=True) 87 | poll_result = queryjob.poll() 88 | for event in poll_result.events: 89 | print(event) 90 | 91 | # With a static queryjob you can poll it iterativly until it has been exhausted 92 | queryjob = client.create_queryjob("Login Attempt Failed", is_live=False) 93 | for poll_result in queryjob.poll_until_done(): 94 | print(poll_result.metadata) 95 | for event in poll_result.events: 96 | print(event) 97 | 98 | HumioIngestClient 99 | ***************** 100 | The HumioIngestClient class is used for ingesting data into Humio. 101 | While the HumioClient can also be used for ingesting data, this is mainly meant for debugging. 102 | 103 | .. code-block:: python 104 | 105 | from humiolib.HumioClient import HumioIngestClient 106 | 107 | # Creating the client 108 | client = HumioIngestClient( 109 | base_url= "https://cloud.humio.com", 110 | ingest_token="*****") 111 | 112 | # Ingesting Unstructured Data 113 | messages = [ 114 | "192.168.1.21 - user1 [02/Nov/2017:13:48:26 +0000] \"POST /humio/api/v1/ingest/elastic-bulk HTTP/1.1\" 200 0 \"-\" \"useragent\" 0.015 664 0.015", 115 | "192.168.1..21 - user2 [02/Nov/2017:13:49:09 +0000] \"POST /humio/api/v1/ingest/elastic-bulk HTTP/1.1\" 200 0 \"-\" \"useragent\" 0.013 565 0.013" 116 | ] 117 | 118 | client.ingest_messages(messages) 119 | 120 | # Ingesting Structured Data 121 | structured_data = [ 122 | { 123 | "tags": {"host": "server1" }, 124 | "events": [ 125 | { 126 | "timestamp": "2020-03-23T00:00:00+00:00", 127 | "attributes": {"key1": "value1", "key2": "value2"} 128 | } 129 | ] 130 | } 131 | ] 132 | 133 | client.ingest_json_data(structured_data) 134 | 135 | -------------------------------------------------------------------------------- /tests/cassettes/queryjob/test_poll_until_done_live_queryjob_non_aggregate_query: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"queryString": "", "isLive": true}' 4 | headers: 5 | Accept: 6 | - '*/*' 7 | Accept-Encoding: 8 | - gzip, deflate 9 | Connection: 10 | - keep-alive 11 | Content-Length: 12 | - '35' 13 | Content-Type: 14 | - application/json 15 | User-Agent: 16 | - python-requests/2.22.0 17 | method: POST 18 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs 19 | response: 20 | body: 21 | string: !!binary | 22 | H4sIAAAAAAAAAKpWykxRslIyMtHNTU93co5McXQyy670ssjNcU/MdS7wd05S0lEqLE0tqvTPC8tM 23 | LQeq1VKqBQAAAP//AwCF7YAcNgAAAA== 24 | headers: 25 | Connection: 26 | - keep-alive 27 | Content-Encoding: 28 | - gzip 29 | Content-Type: 30 | - application/json 31 | Date: 32 | - Wed, 25 Mar 2020 14:37:38 GMT 33 | Server: 34 | - nginx 35 | Strict-Transport-Security: 36 | - max-age=31536000;includeSubDomains 37 | Transfer-Encoding: 38 | - chunked 39 | X-Content-Type-Options: 40 | - nosniff 41 | X-XSS-Protection: 42 | - 1;mode=block 43 | status: 44 | code: 200 45 | message: OK 46 | - request: 47 | body: null 48 | headers: 49 | Accept: 50 | - '*/*' 51 | Accept-Encoding: 52 | - gzip, deflate 53 | Connection: 54 | - keep-alive 55 | Content-Type: 56 | - application/json 57 | User-Agent: 58 | - python-requests/2.22.0 59 | method: GET 60 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs/24-mggBCYdAB6kyJ8mlGamCpOCb 61 | response: 62 | body: 63 | string: !!binary | 64 | H4sIAAAAAAAAAOycW1PjNhTHv0rGfWsh0cXy7ancpg3d7bJcdtkuOxmTiODF2MEXCDB89x4nkZyZ 65 | IkMfIqLBPNlRkO1f/j46f9k6j9YwTIY8jvnICi7COOcb1ihNuBUUWQnb/JYnRW4F3x+t34vomudF 66 | eD2xAsw8hm3GHJ/69ob1S3E/gX+xrm4t2Mn4JIWdPExG5+l0UEaHu/z0xD35VhxMH/r7+3fT6fbN 67 | 9gf46qzLh9nhrH+q/Sy8y4ssSsbwAfZJFzteF3e7BHc2O2XOM9L5jkjv7/S2RxB2A0wD2w+Q3/kN 68 | wd+Pzpl18OnouNO7LK+jtBdOot4t7kFvcNo9Hod5EQ03z8v4qvPn8fFBD3fxmdUhCHUQ/OcmbJ9Z 69 | 1UHCMVwz7KEuwrTDHDbfqs4vAkrWEf32x1b/53g7nY5P93f6117/i98fkgEauIOajPW08ZbQbH/B 70 | DP+XmRdQujpmdsdh7oyZ/RpmzquYea7TCq0WGquheWqhaYFmjNDsVzCzPeoh2kY0GdGogAZkVEKD 71 | JtdzVw7NGKGRVzEjHkErZ2bO0IlraEQtNOLhVmjy5kQvM6MEOYSSdye04c/J3s3Xz18O/zr/NCQT 72 | dnh6no124gEeUJGkVWgUStNFbc1CWgM0kaU1Q2Ou/f4GzwZqIk0DakwtNS3UzJGayNOaoCHme9Rv 73 | o1od1USiRgGNSmqaqJkjNZGpNUPzXOa1UqulJlI1oKYyn1WTDmrmSE3kao3QKHOd92cK1AMo8RcZ 74 | LkVUOYBCkw5qxkiNeK+Dhj3cRjUZ1Yi0Bc1S00HNHKlJW6CGRnwf5m5bW1A7UCJsQYVGkavpomaO 75 | 1IQteAEaRqyNanVUE7ZgDaiZIzVhC5qgeZ6HaZurLUU1YQsIoFFFNU3UzJGasAXN0FzXbqNaLTUs 76 | bAFQUw6g0KSDmjFSw8IWNEFjvuM77dOCJakJW0AAjSqqaaJmjtSELXgBmo3aebUlqUlb0Cw1HdTM 77 | kZq0BU3QmOOy1T83XreH7ep5NSxtAbzDp4xqeqiZIzVpCxqhMddvJzuWopq0BYyppnAJNOmgZo7U 78 | pC1ogEZtz7XbAbSWmnQFQEYV1DRBM0Zp0hQ0MaMOIa0pqIUmPQGlytETmnRAM0Zo0hI0MEMuZe/w 79 | pUh1niYdAZBRRTRN0IwRmjQETcyYi5x26KwjmvQDiCkn1KBJBzRjhCbtgJoZdhjz7fYpwZIdWLyb 80 | UJFRRDRd0IwRmjADDcyQ4/rEbac4hNAAmUjSKjQKpemiZojSAJrI0hqhMdtjbUxbkppI05DDbKXU 81 | 9FAzR2oiT2uCBmukbNtZ+fsczFavlfUDjFa3VpbBWllYfwxrZVm9VlblB+D+FHkaAjIqpWmCZsps 82 | N1ATmdoaUDPn/hTztmsATa7/f3YtO3FWen869v+4P0Wq1giN2gSvfu2KMUFNTNsiQpWjJzTpYGZO 83 | TBPztmsAzZiQJh3B2wvNmIgmDUETMyh6gla/lsCYgCbtAIGs7vkqMAiadDAzJ6BJO/D20IwJaLUb 84 | eHOhGRPQai+gYkZ9hKAqVPsSx9IMx3zWdkbm+Yi2gLb6UcCYm3NhBRbMfmxY17wId8MitILHeV21 85 | nbRMCiuoClvxaZHJxssw/5hmfG9Re82a1WUD7NZFFBc8+1zy7L7q5KbaOBL10n6FCYQo7yfwjXBY 86 | RLdQVG1R0C1KhnE54rs85gUfiW4XjTyBmmbzqmUuLNplFH7C/DK9mx1l9t3dqKrJdl4WUZrUfeYf 87 | lg8BdeEyuJSqH+TAjM68nyQ95HkZFydJEcW7szpvs6PCpUT51nic8XFY1Oc5SeN46wLO3wqIZ4ND 88 | mmTpkOc5H23fFxyq0GHkYZgNkh+LS6kIzmDsPXMxC0rPnV82O7vt8uKCZ0fRA5xJ1VNVlO5jFMdR 89 | dUSIAkVahPHXNLuC09qw7sIsgZJyVU08+E3v4OP5hZGnp38BAAD//wMAbgt85E1PAAA= 90 | headers: 91 | Connection: 92 | - keep-alive 93 | Content-Encoding: 94 | - gzip 95 | Content-Type: 96 | - application/json 97 | Date: 98 | - Wed, 25 Mar 2020 14:37:38 GMT 99 | Server: 100 | - nginx 101 | Strict-Transport-Security: 102 | - max-age=31536000;includeSubDomains 103 | Transfer-Encoding: 104 | - chunked 105 | X-Content-Type-Options: 106 | - nosniff 107 | X-XSS-Protection: 108 | - 1;mode=block 109 | status: 110 | code: 200 111 | message: OK 112 | - request: 113 | body: null 114 | headers: 115 | Accept: 116 | - '*/*' 117 | Accept-Encoding: 118 | - gzip, deflate 119 | Connection: 120 | - keep-alive 121 | Content-Length: 122 | - '0' 123 | Content-Type: 124 | - application/json 125 | User-Agent: 126 | - python-requests/2.22.0 127 | method: DELETE 128 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs/24-mggBCYdAB6kyJ8mlGamCpOCb 129 | response: 130 | body: 131 | string: '' 132 | headers: 133 | Connection: 134 | - keep-alive 135 | Date: 136 | - Wed, 25 Mar 2020 14:37:38 GMT 137 | Server: 138 | - nginx 139 | Strict-Transport-Security: 140 | - max-age=31536000;includeSubDomains 141 | X-Content-Type-Options: 142 | - nosniff 143 | X-XSS-Protection: 144 | - 1;mode=block 145 | status: 146 | code: 204 147 | message: No Content 148 | version: 1 149 | -------------------------------------------------------------------------------- /tests/cassettes/queryjob/test_poll_until_done_live_queryjob_after_fully_polled_fail: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"queryString": "timechart()", "isLive": true}' 4 | headers: 5 | Accept: 6 | - '*/*' 7 | Accept-Encoding: 8 | - gzip, deflate 9 | Connection: 10 | - keep-alive 11 | Content-Length: 12 | - '46' 13 | Content-Type: 14 | - application/json 15 | User-Agent: 16 | - python-requests/2.22.0 17 | method: POST 18 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs 19 | response: 20 | body: 21 | string: !!binary | 22 | H4sIAAAAAAAAAKpWykxRslIyMtHNz0tzdbN0cQssTzb3rEhOscjO9nf1Ng5S0lEqLE0tqvTPC8tM 23 | LQeqLcnMTU3OSCwq0dBUqgUAAAD//wMAXq0t9UAAAAA= 24 | headers: 25 | Connection: 26 | - keep-alive 27 | Content-Encoding: 28 | - gzip 29 | Content-Type: 30 | - application/json 31 | Date: 32 | - Wed, 25 Mar 2020 14:24:22 GMT 33 | Server: 34 | - nginx 35 | Strict-Transport-Security: 36 | - max-age=31536000;includeSubDomains 37 | Transfer-Encoding: 38 | - chunked 39 | X-Content-Type-Options: 40 | - nosniff 41 | X-XSS-Protection: 42 | - 1;mode=block 43 | status: 44 | code: 200 45 | message: OK 46 | - request: 47 | body: null 48 | headers: 49 | Accept: 50 | - '*/*' 51 | Accept-Encoding: 52 | - gzip, deflate 53 | Connection: 54 | - keep-alive 55 | Content-Type: 56 | - application/json 57 | User-Agent: 58 | - python-requests/2.22.0 59 | method: GET 60 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs/24-onfEF9DFQwc7Ixcd8kkOEK3R 61 | response: 62 | body: 63 | string: !!binary | 64 | H4sIAAAAAAAAAIzYXW/TMBQG4L+CfIk6ycd2Yjt3G90FElzAhLhAqMpat7OWJiNxVsbU/47zsW5j 65 | EXl71Y8j+9hPnPScR7bOy7UrCrdh2TYvGrdgm6p0LAt1G9+7e1eGhmU/HtlqXbVlYBnjbMFW1+36 66 | 1nWfKDEJT6zk3YsdF/8PTLkAA4nAwGFAYGphwRGlAQOVBgOTFAxMEzBQKzDQoDIWlNEclNFD3LyM 67 | JlBGC1BGS1BGK1BGJ6CMTkEZrUEZbVAZC8oMuwgcV8NBGUOgjBGgjJGgjFGgjElAGZOCMkaDMsag 68 | MsN+z58ZY0EZy0EZS6CMFaCMlaCMVaCMTUAZm4IydrxLzT657HhogEBUZhxybsR46WAyxAmTIS4w 69 | GeISkyGuMBniCSZDPMXODI27M3tmYiAmQ9xgZ4a4BWXiaqDnNRGBMiRAGZKgDClQhhJQZrwc52Uo 70 | BWVIgzJkUBkLyggOyggCZYQAZYQEZYQCZcbzPy8jElBGpKCM0KCMMKiMBWUkB2UkgTJSgDJSgjLj 71 | DXdeRipQRiagjExBGamnZMSbupBkrA271+tHHE1F2ika9XZIxado0olAmqIRb6vXeN+bspnKcnzI 72 | zeMoOYVjJrJUII5KnnB+LtjehXyZh5xlj0OJ/mGoza2OJfvvUJ9+DH7v1jd53VXqXUUfExgq91WR 73 | N+F1FU+nKU5BzV1erm7afV76P12HIFb77/a+bINrnkfqg/a+KHxsFLBx3QvW+qxpdzvXBLc5O/jN 74 | rm8XdAmdDRmdZtn6+t9cXnYU2NYXwdVfWlc/dAv+1b25CrUvd3HA9zER33wsY0S+Dv4+ti7GPoYv 75 | 10W7cUtXuJjC5djJGH90ZVxO17sglYo0Pn/UgjU31aGfpY9d+ibOcd0GX5XPYzafXk7RhH5rxx6I 76 | Gccpq6+uaYvwrQy+WPbdlH7WY5fp+W5Xu10eTi2Wu6oozrcxfZbFm0lM466u1q5p3ObiodvnLP4d 77 | ofTF108rSWNsvxeXE2sZN2kqvbpP7qLdbl19FV1Z1l02HcznEZGS+LkKefG9qm9jWgt2yOsy7nfX 78 | CYqX3yF+PaxLHI9/AQAA//8DAKzyeEJDEgAA 79 | headers: 80 | Connection: 81 | - keep-alive 82 | Content-Encoding: 83 | - gzip 84 | Content-Type: 85 | - application/json 86 | Date: 87 | - Wed, 25 Mar 2020 14:24:22 GMT 88 | Server: 89 | - nginx 90 | Strict-Transport-Security: 91 | - max-age=31536000;includeSubDomains 92 | Transfer-Encoding: 93 | - chunked 94 | X-Content-Type-Options: 95 | - nosniff 96 | X-XSS-Protection: 97 | - 1;mode=block 98 | status: 99 | code: 200 100 | message: OK 101 | - request: 102 | body: null 103 | headers: 104 | Accept: 105 | - '*/*' 106 | Accept-Encoding: 107 | - gzip, deflate 108 | Connection: 109 | - keep-alive 110 | Content-Type: 111 | - application/json 112 | User-Agent: 113 | - python-requests/2.22.0 114 | method: GET 115 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs/24-onfEF9DFQwc7Ixcd8kkOEK3R 116 | response: 117 | body: 118 | string: !!binary | 119 | H4sIAAAAAAAAAIzYXW+bMBQG4L8y+XKiko9tsM1du/Ri0naxVdMupimixElRCXRgmnVV/vvMR9N2 120 | RePNVT6O7GM/GHLOI8uzKndl6TYs3WZl6yK2qSvHUt904b27d5VvWfrjka3zuqs8SxlnEVtfd/mt 121 | 6z9RbGIeW8n7FztG/w9MuAADicDAcUBgamHBEaUBA5UGA+MEDExiMFArMNCgMhaU0RyU0WPcsowm 122 | UEYLUEZLUEYrUEbHoIxOQBmtQRltUBkLyoy7CBxXw0EZQ6CMEaCMkaCMUaCMiUEZk4AyRoMyxqAy 123 | 434vnxljQRnLQRlLoIwVoIyVoIxVoIyNQRmbgDJ2ukstPrnsdGiAQFRmGnJpxHDpYDLECZMhLjAZ 124 | 4hKTIa4wGeIxJkM8wc4MTbuzeGZCICZD3GBnhrgFZcJqoOc1EYEyJEAZkqAMKVCGYlBmuhyXZSgB 125 | ZUiDMmRQGQvKCA7KCAJlhABlhARlhAJlpvO/LCNiUEYkoIzQoIwwqIwFZSQHZSSBMlKAMlKCMtMN 126 | d1lGKlBGxqCMTEAZqedkxJu6kGSoDfvX60cczUXaORr1dkjF52iSmUCaoxFvq9dw35uzmctyesgt 127 | 4yg5h2NmslQgjoqfcH5GbO98tsp8xtLHsUT/MNbmVoeS/bdvTj/6Yu/ym6zpK/W+og8JjJX7usxa 128 | /7qKp9MUp6D2LqvWN90+q4o/fYcgVPvv9kXVedc+jzQE7YuyLEKjgE3rjlhXpG2327nWu83Zodjs 129 | hnZBn9DZmNFplm3R/JvLy44C2xald82XzjUP/YJ/9W+ufFNUuzDg+5BI0X6sQkSW++I+tC6mPkZR 130 | 5WW3cStXupDC5dTJmH50VVhO37sglYhExSQi1t7Uh2GWIXZVtGGO684XdfU8Zvvp5RStH7Z26oGY 131 | aZyq/urarvTfKl+Uq6GbMsx67DM93+0at8v8qcVyV5fl+Takz1IRUxyxu6bOXdu6zcVDv89p+DtC 132 | yYuvn1aSqGjci8uZtUybNJdeMyR30W23rrkKriztL5se5vOEKIwM2+Frn5Xf6+Y2JBaxQ9ZUYcf7 133 | XlC4AA/h63Fl4nj8CwAA//8DAMP9JvxFEgAA 134 | headers: 135 | Connection: 136 | - keep-alive 137 | Content-Encoding: 138 | - gzip 139 | Content-Type: 140 | - application/json 141 | Date: 142 | - Wed, 25 Mar 2020 14:24:25 GMT 143 | Server: 144 | - nginx 145 | Strict-Transport-Security: 146 | - max-age=31536000;includeSubDomains 147 | Transfer-Encoding: 148 | - chunked 149 | X-Content-Type-Options: 150 | - nosniff 151 | X-XSS-Protection: 152 | - 1;mode=block 153 | status: 154 | code: 200 155 | message: OK 156 | version: 1 157 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Contributing 3 | ============ 4 | Contributions are welcome, and they are greatly appreciated! 5 | Every little bit helps, and credit will always be given. 6 | 7 | Ways To Contribute 8 | ================== 9 | There are many different ways, in which you may contribute to this project, including: 10 | 11 | * Opening issues by using the `issue tracker `_, using the correct issue template for your submission. 12 | * Commenting and expanding on open issues. 13 | * Propose fixes to open issues via a pull request. 14 | 15 | We suggest that you create an issue on GitHub before starting to work on a pull request, as this gives us a better overview, and allows us to start a conversation about the issue. 16 | We also encourage you to separate unrelated contributions into different pull requests. This makes it easier for us to understand your individual contributions and faster at reviewing them. 17 | 18 | Setting Up `humiolib` For Local Development 19 | =========================================== 20 | 21 | 1. Fork `python-humio `_ 22 | (look for the "Fork" button). 23 | 2. Clone your fork locally:: 24 | 25 | git clone git@github.com/humio/python-humio.git 26 | 27 | 3. Create a branch for local development:: 28 | 29 | git checkout -b name-of-your-bugfix-or-feature 30 | 31 | 4. Install `humiolib` from your local repository:: 32 | 33 | pip install -e . 34 | 35 | Now you can import `humiolib` into your Python code, and you can make changes to the project locally. 36 | 37 | 5. As your work progresses, regularly commit to and push your branch to your own fork on GitHub:: 38 | 39 | git add . 40 | git commit -m "Your detailed description of your changes." 41 | git push origin name-of-your-bugfix-or-feature 42 | 43 | 44 | Running Tests locally 45 | ===================== 46 | Testing is accomplished using the `pytest `_ library. This should automatically be installed on your machine, when you install the `humiolib` package. 47 | To run tests simply execute the following command in the `tests` folder: 48 | 49 | .. code-block:: bash 50 | 51 | pytest 52 | 53 | Humio API calls made during tests have been recorded using `vcr.py `_ and can be found in the `tests/cassettes` folder. 54 | These will be *played back* when tests are run, so you do not need to set up a Humio instance to perform the tests. 55 | Please do not re-record cassettes unless you're really familiar with vcr.py. 56 | 57 | 58 | Building Documentation From Source 59 | =================================== 60 | If you're contributing to the documentation, you need to build the docs locally to inspect your changes. 61 | 62 | To do this, first make sure you have the documentation dependencies installed:: 63 | 64 | pip install -r docs/requirements.txt 65 | 66 | Once dependencies have been installed build the HTML pages using sphinx:: 67 | 68 | sphinx-build -b html docs build/docs 69 | 70 | You should now find the generated HTML in ``build/docs``. 71 | 72 | 73 | Making A Pull Request 74 | ===================== 75 | When you have made your changes locally, or you want feedback on a work in progress, you're almost ready to make a pull request. 76 | 77 | If you have changed part of the codebase in your pull request, please go through this checklist: 78 | 79 | 1. Write new test cases if the old ones do not cover your new code. 80 | 2. Update documentation if necessary. 81 | 3. Add yourself to ``AUTHORS.rst``. 82 | 83 | If you have only changed the documentation you only need to add yourself to ``AUTHORS.rst``. 84 | 85 | When you've been through the applicable checklist, push your final changes to your development branch on GitHub. 86 | Afterwards, use the GitHub interface to create a pull request to the official repository. 87 | 88 | 89 | Publishing the Library to PyPI 90 | ============================== 91 | This section describes the manual process of publishing this library to PyPI. 92 | This is a task only done by maintainers of the repository, and it is always done from the ``master`` branch. 93 | 94 | Before the package can be published, you need to bump the semantic version of the library. This is done using the program ``bump2version``, which can be installed as such: 95 | 96 | .. code-block:: bash 97 | 98 | pip3 install bump2version 99 | 100 | You can now bump the library to either a new patch, minor or major version, using the following command: 101 | 102 | .. code-block:: bash 103 | 104 | bumpversion (patch | minor | major) 105 | 106 | This will bump the version across library as specified in ``.bumpversion.cfg``. 107 | 108 | Once the version has been bumped, add a descriptive entry to ``CHANGELOG.rst`` about what has changed in the new version of the library. 109 | 110 | You will not need to change any more tracked files during the publishing process, so create a new commit to encompass the changes made by your version bump now. 111 | 112 | To build the library into a package run: 113 | 114 | .. code-block:: bash 115 | 116 | python3 setup.py bdist_wheel sdist 117 | 118 | This will create a build and source distribution of the library within the ``/dist`` folder. 119 | 120 | To upload these files to PyPI you need to install ``twine``, which can be done using the following command: 121 | 122 | .. code-block:: bash 123 | 124 | pip3 install twine 125 | 126 | Now upload the contents of ``/dist`` to PyPI by entering the following command and following the prompt on the screen: 127 | 128 | .. code-block:: bash 129 | 130 | twine upload dist/* 131 | 132 | Congratulations! The new version of the package should now be live on PyPI for all to enjoy. 133 | 134 | Terms of Service For Contributors 135 | ================================= 136 | For all contributions to this repository (software, bug fixes, configuration changes, documentation, or any other materials), we emphasize that this happens under GitHubs general Terms of Service and the license of this repository. 137 | 138 | Contributing as an individual 139 | ***************************** 140 | If you are contributing as an individual you must make sure to adhere to: 141 | 142 | The `GitHub Terms of Service `_ **Section D. User-Generated Content,** `Subsection: 6. Contributions Under Repository License `_ : 143 | 144 | *Whenever you make a contribution to a repository containing notice of a license, you license your contribution under the same terms, and you agree that you have the right to license your contribution under those terms. If you have a separate agreement to license your contributions under different terms, such as a contributor license agreement, that agreement will supersede. 145 | Isn't this just how it works already? Yep. This is widely accepted as the norm in the open-source community; it's commonly referred to by the shorthand "inbound=outbound". We're just making it explicit."* 146 | 147 | Contributing on behalf of a Corporation 148 | *************************************** 149 | If you are contributing on behalf of a Corporation you must make sure to adhere to: 150 | 151 | The `GitHub Corporate Terms of Service `_ **Section D. Content Responsibility; Ownership; License Rights,** `subsection 5. Contributions Under Repository License `_: 152 | 153 | *Whenever Customer makes a contribution to a repository containing notice of a license, it licenses such contributions under the same terms and agrees that it has the right to license such contributions under those terms. If Customer has a separate agreement to license its contributions under different terms, such as a contributor license agreement, that agreement will supersede* -------------------------------------------------------------------------------- /src/humiolib/QueryJob.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import time 3 | from humiolib.HumioExceptions import HumioQueryJobExhaustedException, HumioHTTPException, HumioQueryJobExpiredException 4 | from humiolib.WebCaller import WebCaller 5 | 6 | class PollResult(): 7 | """ 8 | Result of polling segments of queryjob results. 9 | We choose to return these clusters of data, rather than just a list of events, 10 | as the metadata returned changes between polls. 11 | """ 12 | def __init__(self, events, metadata): 13 | self.events = events 14 | self.metadata = metadata 15 | 16 | 17 | class BaseQueryJob(): 18 | """ 19 | Base QueryJob class, not meant to be instantiated. 20 | This class and its children manage access to queryjobs created on a Humio instance, 21 | they are mainly used for extracting results from queryjobs. 22 | """ 23 | def __init__(self, query_id, base_url, repository, user_token): 24 | """ 25 | Parameters: 26 | query_id (string): Id of queryjob 27 | base_url (string): Url of Humio instance 28 | repository (string): Repository being queried 29 | user_token (string): Token used to access resource 30 | """ 31 | self.query_id = query_id 32 | self.segment_is_done = False 33 | self.segment_is_cancelled = False 34 | self.more_segments_can_be_polled = True 35 | self.time_at_last_poll = 0 36 | self.wait_time_until_next_poll = 0 37 | self.base_url = base_url 38 | self.repository = repository 39 | self.user_token = user_token 40 | self.webcaller = WebCaller(self.base_url) 41 | 42 | @property 43 | def _default_user_headers(self): 44 | """ 45 | :return: Default headers used for web requests 46 | :rtype: dict 47 | """ 48 | return { 49 | "Content-Type": "application/json", 50 | "Authorization": "Bearer {}".format(self.user_token), 51 | } 52 | 53 | 54 | def _wait_till_next_poll(self): 55 | """ 56 | A potentially blocking operation, that waits until the queryjob may be polled again. 57 | This will always pass on the first poll to the queryjob. 58 | """ 59 | time_since_last_poll = time.time() - self.time_at_last_poll 60 | if(time_since_last_poll < self.wait_time_until_next_poll): 61 | time.sleep((self.wait_time_until_next_poll - time_since_last_poll) / 1000.0) 62 | 63 | def _fetch_next_segment(self, link, headers, **kwargs): 64 | """ 65 | Polls the queryjob for the next segment of data. 66 | May block, if the queryjob is not ready to be polled again. 67 | 68 | :param link: url to access queryjob. 69 | :type link: str 70 | :param headers: headers used for web request. 71 | :type headers: list(dict) 72 | 73 | :return: A data object that contains events of the polled segment and metadata about the poll 74 | :rtype: PollResult 75 | """ 76 | self._wait_till_next_poll() 77 | 78 | try: 79 | response = self.webcaller.call_rest("get", link, headers=headers, **kwargs).json() 80 | except HumioHTTPException as e: 81 | # In the case that the queryjob has expired, a custom exception is thrown. 82 | # The calling code must itself decide how to respond to the error. 83 | # It has been considered whether this instance should simply restart the queryjob automatically, 84 | # but that would require the calling code to handle cases where 85 | # a queryjob restart returns previously received query results. 86 | if e.status_code == 404: 87 | raise HumioQueryJobExpiredException(e.message) 88 | else: 89 | raise e 90 | 91 | self.wait_time_until_next_poll = response["metaData"]["pollAfter"] 92 | self.segment_is_done = response["done"] 93 | self.segment_is_cancelled = response["cancelled"] 94 | self.time_at_last_poll = time.time() 95 | 96 | return PollResult(response["events"], response["metaData"]) 97 | 98 | def _is_streaming_query(self, metadata): 99 | """ 100 | Checks whether the query is a streaming query and not an aggregate 101 | 102 | :param metaData: query response metadata. 103 | :type metadata: dict 104 | 105 | :return: Answer to whether query is of type streaming 106 | :rtype: Bool 107 | """ 108 | return not metadata["isAggregate"] 109 | 110 | def poll(self, **kwargs): 111 | """ 112 | Polls the queryjob for the next segment of data, and handles edge cases for data polled 113 | 114 | :return: A data object that contains events of the polled segment and metadata about the poll 115 | :rtype: PollResult 116 | """ 117 | link = "dataspaces/{}/queryjobs/{}".format(self.repository, self.query_id) 118 | 119 | headers = self._default_user_headers 120 | headers.update(kwargs.pop("headers", {})) 121 | 122 | poll_result = self._fetch_next_segment(link, headers, **kwargs) 123 | while not self.segment_is_done: # In case the segment hasn't been completed, we poll until is is 124 | poll_result = self._fetch_next_segment(link, headers, **kwargs) 125 | 126 | if self._is_streaming_query(poll_result.metadata): 127 | self.more_segments_can_be_polled = poll_result.metadata["extraData"]["hasMoreEvents"] == 'true' 128 | else: # is aggregate query 129 | self.more_segments_can_be_polled = False 130 | 131 | return poll_result 132 | 133 | 134 | class StaticQueryJob(BaseQueryJob): 135 | """ 136 | Manages a static queryjob 137 | """ 138 | def __init__(self, query_id, base_url, repository, user_token): 139 | """ 140 | :param query_id: Id of queryjob. 141 | :type query_id: str 142 | :param base_url: Url of Humio instance. 143 | :type base_url: str 144 | :param repository: Repository being queried. 145 | :type repository: str 146 | :param user_token: Token used to access resource. 147 | :type user_token: str 148 | """ 149 | super().__init__(query_id, base_url, repository, user_token) 150 | 151 | def poll(self, **kwargs): 152 | """ 153 | Polls next segment of result 154 | 155 | :return: A data object that contains events of the polled segment and metadata about the poll 156 | :rtype: PollResult 157 | """ 158 | if not self.more_segments_can_be_polled: 159 | raise HumioQueryJobExhaustedException() 160 | 161 | return super().poll(**kwargs) 162 | 163 | def poll_until_done(self, **kwargs): 164 | """ 165 | Create generator for yielding poll results 166 | 167 | :return: A generator for query results 168 | :rtype: Generator 169 | """ 170 | 171 | yield self.poll(**kwargs) 172 | while self.more_segments_can_be_polled: 173 | yield self.poll(**kwargs) 174 | 175 | 176 | class LiveQueryJob(BaseQueryJob): 177 | """ 178 | Manages a live queryjob 179 | """ 180 | def __init__(self, query_id, base_url, repository, user_token): 181 | """ 182 | :param query_id: Id of queryjob. 183 | :type query_id: str 184 | :param base_url: Url of Humio instance. 185 | :type base_url: str 186 | :param repository: Repository being queried. 187 | :type repository: str 188 | :param user_token: Token used to access resource. 189 | :type user_token: str 190 | """ 191 | super().__init__(query_id, base_url, repository, user_token) 192 | 193 | def __del__(self): 194 | """ 195 | Delete queryjob, when this object is deconstructed. 196 | As live queryjobs are kept around for 1 hours after last query, 197 | it'd be best to delete them when not in use. 198 | """ 199 | try: 200 | headers = self._default_user_headers 201 | endpoint = "dataspaces/{}/queryjobs/{}".format(self.repository, self.query_id) 202 | self.webcaller.call_rest("delete", endpoint, headers) 203 | except HumioHTTPException: # If the queryjob doesn't exists anymore, we don't want to halt on the exception 204 | pass 205 | -------------------------------------------------------------------------------- /tests/cassettes/queryjob/test_poll_until_done_live_queryjob_poll_after_done_success: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"queryString": "timechart()", "isLive": true}' 4 | headers: 5 | Accept: 6 | - '*/*' 7 | Accept-Encoding: 8 | - gzip, deflate 9 | Connection: 10 | - keep-alive 11 | Content-Length: 12 | - '46' 13 | Content-Type: 14 | - application/json 15 | User-Agent: 16 | - python-requests/2.22.0 17 | method: POST 18 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs 19 | response: 20 | body: 21 | string: !!binary | 22 | H4sIAAAAAAAAAKpWykxRslIyMtHNN3Er8UxOKihNrkj3DE2qcExxDjbO8whS0lEqLE0tqvTPC8tM 23 | LQeqLcnMTU3OSCwq0dBUqgUAAAD//wMAuRg2OUAAAAA= 24 | headers: 25 | Connection: 26 | - keep-alive 27 | Content-Encoding: 28 | - gzip 29 | Content-Type: 30 | - application/json 31 | Date: 32 | - Wed, 25 Mar 2020 14:26:16 GMT 33 | Server: 34 | - nginx 35 | Strict-Transport-Security: 36 | - max-age=31536000;includeSubDomains 37 | Transfer-Encoding: 38 | - chunked 39 | X-Content-Type-Options: 40 | - nosniff 41 | X-XSS-Protection: 42 | - 1;mode=block 43 | status: 44 | code: 200 45 | message: OK 46 | - request: 47 | body: null 48 | headers: 49 | Accept: 50 | - '*/*' 51 | Accept-Encoding: 52 | - gzip, deflate 53 | Connection: 54 | - keep-alive 55 | Content-Type: 56 | - application/json 57 | User-Agent: 58 | - python-requests/2.22.0 59 | method: GET 60 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs/24-o4FtIcbpucxgIUbxAdCS3nHR 61 | response: 62 | body: 63 | string: !!binary | 64 | H4sIAAAAAAAAAGyST0/DMAzFvwryERVpZRtjvW1sByQ4AEIcEJpC63bW0gQSh/FH++44bdchtJ5S 65 | x7J/7738QK5MjlpjAVmptMcECmuw/8EPNOwhe35JoEZWC8UKsh9o6lc2GIZskAB+suvvmGrM18rJ 66 | FbALCAm8hnyDvNLK86o9y106vhyno/FkEL9Dk39TZrUOtTL0HbGk76QmExj9v6aatCaBg+l+RKDM 67 | h6pCz1icbamosIEQoLOWqEcpyf1nGYynw3bQLoGSNKO7C+i+ot73eHhgR6aSgacCQv7aSIfKmT4O 68 | fpHJdShwgRoFYdm51zmLRuS0qi+Gk4t0mCbg13bbbGl6F+Rlx2tgsqbPgPzN3xWeG2vjHCGednOM 69 | vUcfND8aJr04RChSyM+qymGlWDhjIAm8Wa1npeBDdj65HEnB2Ry9x2L+FX2OmfalvQqpNTYsj8jo 70 | /DlG5hqueShLdA8SaTM8RnLbxZeKDWxZ6SfrNgKUwFY5I053724r5VbRYLf7BQAA//8DAKAxy6Wy 71 | AgAA 72 | headers: 73 | Connection: 74 | - keep-alive 75 | Content-Encoding: 76 | - gzip 77 | Content-Type: 78 | - application/json 79 | Date: 80 | - Wed, 25 Mar 2020 14:26:16 GMT 81 | Server: 82 | - nginx 83 | Strict-Transport-Security: 84 | - max-age=31536000;includeSubDomains 85 | Transfer-Encoding: 86 | - chunked 87 | X-Content-Type-Options: 88 | - nosniff 89 | X-XSS-Protection: 90 | - 1;mode=block 91 | status: 92 | code: 200 93 | message: OK 94 | - request: 95 | body: null 96 | headers: 97 | Accept: 98 | - '*/*' 99 | Accept-Encoding: 100 | - gzip, deflate 101 | Connection: 102 | - keep-alive 103 | Content-Type: 104 | - application/json 105 | User-Agent: 106 | - python-requests/2.22.0 107 | method: GET 108 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs/24-o4FtIcbpucxgIUbxAdCS3nHR 109 | response: 110 | body: 111 | string: !!binary | 112 | H4sIAAAAAAAAAIzYz2+bMBQH8H9l8nGikp9tsM2tXXqYtB22atphmiJKnBSVQAemWVflf5/50bRd 113 | 0fjmlJAn+9kfHPLeI8uzKndl6TYs3WZl6yK2qSvHUt904b27d5VvWfrjka3zuqs8SxlnEVtfd/mt 114 | 6z9RbGIeW8n7FztG/w9MuAADicDAcUBgamHBEaUBA5UGA+MEDExiMFArMNCgMhaU0RyU0WPcsowm 115 | UEYLUEZLUEYrUEbHoIxOQBmtQRltUBkLyoy7CBxXw0EZQ6CMEaCMkaCMUaCMiUEZk4AyRoMyxqAy 116 | 434vnxljQRnLQRlLoIwVoIyVoIxVoIyNQRmbgDJ2+pVafHLZ6dAAgajMNOTSiOHWwWSIEyZDXGAy 117 | xCUmQ1xhMsRjTIZ4gp0ZmnZn8cyEQEyGuMHODHELyoTVQM9rIgJlSIAyJEEZUqAMxaDMdDsuy1AC 118 | ypAGZcigMhaUERyUEQTKCAHKCAnKCAXKTOd/WUbEoIxIQBmhQRlhUBkLykgOykgCZaQAZaQEZaYf 119 | 3GUZqUAZGYMyMgFlpJ6TEW/qQpKhNuxfrx9xNBdp52jU2yEVn6NJZgJpjka8rV7D796czVyW00Nu 120 | GUfJORwzk6UCcVT8hPMzYnvns1XmM5Y+jiX6h7E2tzqU7L99c/rSF3uX32RNX6n3FX1IYKzc12XW 121 | +tdVPJ2mOAW1d1m1vun2WVX86TsEodp/ty+qzrv2eaQhaF+UZREaBWxad8S6Im273c613m3ODsVm 122 | N7QL+oTOxoxOs2yL5t9cXnYU2LYovWu+dK556Bf8q39z5Zui2oUB34dEivZjFSKy3Bf3oXUx9TGK 123 | Ki+7jVu50oUULqdOxvSlq8Jy+t4FqUTq8G9aRKy9qQ/DLEPsqmjDHNedL+rqecz208spWj9s7dQD 124 | sdM4Vf3VtV3pv1W+KFdDN2WY9dhner7bNW6X+VOL5a4uy/NtSJ+lIk4oYndNnbu2dZuLh36f0/B3 125 | hJIXl59Wkqho3IvLmbVMmzSXXjMkd9Ftt665Cq4s7W+bHubzhChCYROu1D4rv9fNbUgsYoesqcKO 126 | 972gcAMewuVxZeJ4/AsAAP//AwDS4pvYRRIAAA== 127 | headers: 128 | Connection: 129 | - keep-alive 130 | Content-Encoding: 131 | - gzip 132 | Content-Type: 133 | - application/json 134 | Date: 135 | - Wed, 25 Mar 2020 14:26:19 GMT 136 | Server: 137 | - nginx 138 | Strict-Transport-Security: 139 | - max-age=31536000;includeSubDomains 140 | Transfer-Encoding: 141 | - chunked 142 | X-Content-Type-Options: 143 | - nosniff 144 | X-XSS-Protection: 145 | - 1;mode=block 146 | status: 147 | code: 200 148 | message: OK 149 | - request: 150 | body: null 151 | headers: 152 | Accept: 153 | - '*/*' 154 | Accept-Encoding: 155 | - gzip, deflate 156 | Connection: 157 | - keep-alive 158 | Content-Type: 159 | - application/json 160 | User-Agent: 161 | - python-requests/2.22.0 162 | method: GET 163 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs/24-o4FtIcbpucxgIUbxAdCS3nHR 164 | response: 165 | body: 166 | string: !!binary | 167 | H4sIAAAAAAAAAIzYXW+bMBQG4L8y+XKiko9tsM1du/Ri0naxVdMupimixElRCXRgmnVV/vvMR9N2 168 | RePNVT6O7GM/GHLOI8uzKndl6TYs3WZl6yK2qSvHUt904b27d5VvWfrjka3zuqs8SxlnEVtfd/mt 169 | 6z9RbGIeW8n7FztG/w9MuAADicDAcUBgamHBEaUBA5UGA+MEDExiMFArMNCgMhaU0RyU0WPcsowm 170 | UEYLUEZLUEYrUEbHoIxOQBmtQRltUBkLyoy7CBxXw0EZQ6CMEaCMkaCMUaCMiUEZk4AyRoMyxqAy 171 | 434vnxljQRnLQRlLoIwVoIyVoIxVoIyNQRmbgDJ2ukstPrnsdGiAQFRmGnJpxHDpYDLECZMhLjAZ 172 | 4hKTIa4wGeIxJkM8wc4MTbuzeGZCICZD3GBnhrgFZcJqoOc1EYEyJEAZkqAMKVCGYlBmuhyXZSgB 173 | ZUiDMmRQGQvKCA7KCAJlhABlhARlhAJlpvO/LCNiUEYkoIzQoIwwqIwFZSQHZSSBMlKAMlKCMtMN 174 | d1lGKlBGxqCMTEAZqedkxJu6kGSoDfvX60cczUXaORr1dkjF52iSmUCaoxFvq9dw35uzmctyesgt 175 | 4yg5h2NmslQgjoqfcH5GbO98tsp8xtLHsUT/MNbmVoeS/bdvTj/6Yu/ym6zpK/W+og8JjJX7usxa 176 | /7qKp9MUp6D2LqvWN90+q4o/fYcgVPvv9kXVedc+jzQE7YuyLEKjgE3rjlhXpG2327nWu83Zodjs 177 | hnZBn9DZmNFplm3R/JvLy44C2xald82XzjUP/YJ/9W+ufFNUuzDg+5BI0X6sQkSW++I+tC6mPkZR 178 | 5WW3cStXupDC5dTJmH50VVhO37sglchQvZCIWHtTH4ZZhthV0YY5rjtf1NXzmO2nl1O0ftjaqQdi 179 | p3Gq+qtru9J/q3xRroZuyjDrsc/0fLdr3C7zpxbLXV2W59uQPktFLClid02du7Z1m4uHfp/T8HeE 180 | khdfP60kUdG4F5cza5k2aS69ZkjuottuXXMVXFnaXzY9zOcJMU5syMPXPiu/181tSCxih6ypwo73 181 | vaBwAR7C1+PKxPH4FwAA//8DAJahmX1FEgAA 182 | headers: 183 | Connection: 184 | - keep-alive 185 | Content-Encoding: 186 | - gzip 187 | Content-Type: 188 | - application/json 189 | Date: 190 | - Wed, 25 Mar 2020 14:26:22 GMT 191 | Server: 192 | - nginx 193 | Strict-Transport-Security: 194 | - max-age=31536000;includeSubDomains 195 | Transfer-Encoding: 196 | - chunked 197 | X-Content-Type-Options: 198 | - nosniff 199 | X-XSS-Protection: 200 | - 1;mode=block 201 | status: 202 | code: 200 203 | message: OK 204 | - request: 205 | body: null 206 | headers: 207 | Accept: 208 | - '*/*' 209 | Accept-Encoding: 210 | - gzip, deflate 211 | Connection: 212 | - keep-alive 213 | Content-Length: 214 | - '0' 215 | Content-Type: 216 | - application/json 217 | User-Agent: 218 | - python-requests/2.22.0 219 | method: DELETE 220 | uri: https://cloud.humio.com/api/v1/dataspaces/sandbox/queryjobs/24-o4FtIcbpucxgIUbxAdCS3nHR 221 | response: 222 | body: 223 | string: '' 224 | headers: 225 | Connection: 226 | - keep-alive 227 | Date: 228 | - Wed, 25 Mar 2020 14:26:22 GMT 229 | Server: 230 | - nginx 231 | Strict-Transport-Security: 232 | - max-age=31536000;includeSubDomains 233 | X-Content-Type-Options: 234 | - nosniff 235 | X-XSS-Protection: 236 | - 1;mode=block 237 | status: 238 | code: 204 239 | message: No Content 240 | version: 1 241 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2020 Humio ApS https://humio.com 2 | 3 | Apache License 4 | Version 2.0, January 2004 5 | http://www.apache.org/licenses/ 6 | 7 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 8 | 9 | 1. Definitions. 10 | 11 | "License" shall mean the terms and conditions for use, reproduction, 12 | and distribution as defined by Sections 1 through 9 of this document. 13 | 14 | "Licensor" shall mean the copyright owner or entity authorized by 15 | the copyright owner that is granting the License. 16 | 17 | "Legal Entity" shall mean the union of the acting entity and all 18 | other entities that control, are controlled by, or are under common 19 | control with that entity. For the purposes of this definition, 20 | "control" means (i) the power, direct or indirect, to cause the 21 | direction or management of such entity, whether by contract or 22 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 23 | outstanding shares, or (iii) beneficial ownership of such entity. 24 | 25 | "You" (or "Your") shall mean an individual or Legal Entity 26 | exercising permissions granted by this License. 27 | 28 | "Source" form shall mean the preferred form for making modifications, 29 | including but not limited to software source code, documentation 30 | source, and configuration files. 31 | 32 | "Object" form shall mean any form resulting from mechanical 33 | transformation or translation of a Source form, including but 34 | not limited to compiled object code, generated documentation, 35 | and conversions to other media types. 36 | 37 | "Work" shall mean the work of authorship, whether in Source or 38 | Object form, made available under the License, as indicated by a 39 | copyright notice that is included in or attached to the work 40 | (an example is provided in the Appendix below). 41 | 42 | "Derivative Works" shall mean any work, whether in Source or Object 43 | form, that is based on (or derived from) the Work and for which the 44 | editorial revisions, annotations, elaborations, or other modifications 45 | represent, as a whole, an original work of authorship. For the purposes 46 | of this License, Derivative Works shall not include works that remain 47 | separable from, or merely link (or bind by name) to the interfaces of, 48 | the Work and Derivative Works thereof. 49 | 50 | "Contribution" shall mean any work of authorship, including 51 | the original version of the Work and any modifications or additions 52 | to that Work or Derivative Works thereof, that is intentionally 53 | submitted to Licensor for inclusion in the Work by the copyright owner 54 | or by an individual or Legal Entity authorized to submit on behalf of 55 | the copyright owner. For the purposes of this definition, "submitted" 56 | means any form of electronic, verbal, or written communication sent 57 | to the Licensor or its representatives, including but not limited to 58 | communication on electronic mailing lists, source code control systems, 59 | and issue tracking systems that are managed by, or on behalf of, the 60 | Licensor for the purpose of discussing and improving the Work, but 61 | excluding communication that is conspicuously marked or otherwise 62 | designated in writing by the copyright owner as "Not a Contribution." 63 | 64 | "Contributor" shall mean Licensor and any individual or Legal Entity 65 | on behalf of whom a Contribution has been received by Licensor and 66 | subsequently incorporated within the Work. 67 | 68 | 2. Grant of Copyright License. Subject to the terms and conditions of 69 | this License, each Contributor hereby grants to You a perpetual, 70 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 71 | copyright license to reproduce, prepare Derivative Works of, 72 | publicly display, publicly perform, sublicense, and distribute the 73 | Work and such Derivative Works in Source or Object form. 74 | 75 | 3. Grant of Patent License. Subject to the terms and conditions of 76 | this License, each Contributor hereby grants to You a perpetual, 77 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 78 | (except as stated in this section) patent license to make, have made, 79 | use, offer to sell, sell, import, and otherwise transfer the Work, 80 | where such license applies only to those patent claims licensable 81 | by such Contributor that are necessarily infringed by their 82 | Contribution(s) alone or by combination of their Contribution(s) 83 | with the Work to which such Contribution(s) was submitted. If You 84 | institute patent litigation against any entity (including a 85 | cross-claim or counterclaim in a lawsuit) alleging that the Work 86 | or a Contribution incorporated within the Work constitutes direct 87 | or contributory patent infringement, then any patent licenses 88 | granted to You under this License for that Work shall terminate 89 | as of the date such litigation is filed. 90 | 91 | 4. Redistribution. You may reproduce and distribute copies of the 92 | Work or Derivative Works thereof in any medium, with or without 93 | modifications, and in Source or Object form, provided that You 94 | meet the following conditions: 95 | 96 | (a) You must give any other recipients of the Work or 97 | Derivative Works a copy of this License; and 98 | 99 | (b) You must cause any modified files to carry prominent notices 100 | stating that You changed the files; and 101 | 102 | (c) You must retain, in the Source form of any Derivative Works 103 | that You distribute, all copyright, patent, trademark, and 104 | attribution notices from the Source form of the Work, 105 | excluding those notices that do not pertain to any part of 106 | the Derivative Works; and 107 | 108 | (d) If the Work includes a "NOTICE" text file as part of its 109 | distribution, then any Derivative Works that You distribute must 110 | include a readable copy of the attribution notices contained 111 | within such NOTICE file, excluding those notices that do not 112 | pertain to any part of the Derivative Works, in at least one 113 | of the following places: within a NOTICE text file distributed 114 | as part of the Derivative Works; within the Source form or 115 | documentation, if provided along with the Derivative Works; or, 116 | within a display generated by the Derivative Works, if and 117 | wherever such third-party notices normally appear. The contents 118 | of the NOTICE file are for informational purposes only and 119 | do not modify the License. You may add Your own attribution 120 | notices within Derivative Works that You distribute, alongside 121 | or as an addendum to the NOTICE text from the Work, provided 122 | that such additional attribution notices cannot be construed 123 | as modifying the License. 124 | 125 | You may add Your own copyright statement to Your modifications and 126 | may provide additional or different license terms and conditions 127 | for use, reproduction, or distribution of Your modifications, or 128 | for any such Derivative Works as a whole, provided Your use, 129 | reproduction, and distribution of the Work otherwise complies with 130 | the conditions stated in this License. 131 | 132 | 5. Submission of Contributions. Unless You explicitly state otherwise, 133 | any Contribution intentionally submitted for inclusion in the Work 134 | by You to the Licensor shall be under the terms and conditions of 135 | this License, without any additional terms or conditions. 136 | Notwithstanding the above, nothing herein shall supersede or modify 137 | the terms of any separate license agreement you may have executed 138 | with Licensor regarding such Contributions. 139 | 140 | 6. Trademarks. This License does not grant permission to use the trade 141 | names, trademarks, service marks, or product names of the Licensor, 142 | except as required for reasonable and customary use in describing the 143 | origin of the Work and reproducing the content of the NOTICE file. 144 | 145 | 7. Disclaimer of Warranty. Unless required by applicable law or 146 | agreed to in writing, Licensor provides the Work (and each 147 | Contributor provides its Contributions) on an "AS IS" BASIS, 148 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 149 | implied, including, without limitation, any warranties or conditions 150 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 151 | PARTICULAR PURPOSE. You are solely responsible for determining the 152 | appropriateness of using or redistributing the Work and assume any 153 | risks associated with Your exercise of permissions under this License. 154 | 155 | 8. Limitation of Liability. In no event and under no legal theory, 156 | whether in tort (including negligence), contract, or otherwise, 157 | unless required by applicable law (such as deliberate and grossly 158 | negligent acts) or agreed to in writing, shall any Contributor be 159 | liable to You for damages, including any direct, indirect, special, 160 | incidental, or consequential damages of any character arising as a 161 | result of this License or out of the use or inability to use the 162 | Work (including but not limited to damages for loss of goodwill, 163 | work stoppage, computer failure or malfunction, or any and all 164 | other commercial damages or losses), even if such Contributor 165 | has been advised of the possibility of such damages. 166 | 167 | 9. Accepting Warranty or Additional Liability. While redistributing 168 | the Work or Derivative Works thereof, You may choose to offer, 169 | and charge a fee for, acceptance of support, warranty, indemnity, 170 | or other liability obligations and/or rights consistent with this 171 | License. However, in accepting such obligations, You may act only 172 | on Your own behalf and on Your sole responsibility, not on behalf 173 | of any other Contributor, and only if You agree to indemnify, 174 | defend, and hold each Contributor harmless for any liability 175 | incurred by, or claims asserted against, such Contributor by reason 176 | of your accepting any such warranty or additional liability. 177 | 178 | END OF TERMS AND CONDITIONS 179 | 180 | APPENDIX: How to apply the Apache License to your work. 181 | 182 | To apply the Apache License to your work, attach the following 183 | boilerplate notice, with the fields enclosed by brackets "[]" 184 | replaced with your own identifying information. (Don't include 185 | the brackets!) The text should be enclosed in the appropriate 186 | comment syntax for the file format. We also recommend that a 187 | file or class name and description of purpose be included on the 188 | same "printed page" as the copyright notice for easier 189 | identification within third-party archives. 190 | 191 | Copyright [yyyy] [name of copyright owner] 192 | 193 | Licensed under the Apache License, Version 2.0 (the "License"); 194 | you may not use this file except in compliance with the License. 195 | You may obtain a copy of the License at 196 | 197 | http://www.apache.org/licenses/LICENSE-2.0 198 | 199 | Unless required by applicable law or agreed to in writing, software 200 | distributed under the License is distributed on an "AS IS" BASIS, 201 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 202 | See the License for the specific language governing permissions and 203 | limitations under the License. 204 | -------------------------------------------------------------------------------- /src/humiolib/HumioClient.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | from humiolib.WebCaller import WebCaller, WebStreamer 4 | from humiolib.QueryJob import StaticQueryJob, LiveQueryJob 5 | from humiolib.HumioExceptions import HumioConnectionException 6 | 7 | 8 | class BaseHumioClient(): 9 | """ 10 | Base class for other client types, is not meant to be instantiated 11 | """ 12 | 13 | def __init__(self, base_url): 14 | self.base_url = base_url 15 | self.webcaller = WebCaller(self.base_url) 16 | 17 | @classmethod 18 | def _from_saved_state(cls, state_dump): 19 | """ 20 | Creates an instance of this class from a saved state 21 | 22 | :param state_dump: json string describing this object. 23 | :type state_dump: str 24 | 25 | :return: An instance of this class 26 | :rtype: BaseHumioClient 27 | """ 28 | data = json.loads(state_dump) 29 | instance = cls(**data) 30 | return instance 31 | 32 | @staticmethod 33 | def _create_unstructured_data_object(messages, parser=None, fields=None, tags=None): 34 | """ 35 | Creates a data object that can be sent to Humio's unstructured ingest endpoints 36 | 37 | :param messages: A list of event strings. 38 | :type messages: list(string) 39 | :param parser: A list of event strings. 40 | :type parser: string, optional 41 | :param fields: Fields that should be added to events after parsing 42 | :type fields: (dict(string->string)), optional 43 | :param tags: Tags to associate with the messages 44 | :type tags: (dict(string->string)), optional 45 | 46 | :return: A data object fit to be sent as unstructured ingest payload 47 | :rtype: dict 48 | """ 49 | return dict( 50 | (k, v) 51 | for k, v in [ 52 | ("messages", messages), 53 | ("type", parser), 54 | ("fields", fields), 55 | ("tags", tags), 56 | ] 57 | if v is not None 58 | ) 59 | 60 | 61 | class HumioClient(BaseHumioClient): 62 | """ 63 | A Humio client that gives full access to the underlying API. 64 | While this client can be used for ingesting data, 65 | we recommend using the HumioIngestClient made exclusivly for ingestion. 66 | """ 67 | 68 | def __init__( 69 | self, 70 | repository, 71 | user_token, 72 | base_url="http://localhost:3000", 73 | ): 74 | """ 75 | :param repository: Repository associated with client 76 | :type repository: str 77 | :param user_token: User token to get access to repository 78 | :type user_token: str 79 | :param base_url: Url of Humio instance 80 | :type repository: str 81 | """ 82 | super().__init__(base_url) 83 | self.repository = repository 84 | self.user_token = user_token 85 | 86 | @property 87 | def _default_user_headers(self): 88 | """ 89 | :return: Default headers used for web requests 90 | :rtype: dict 91 | """ 92 | return { 93 | "Content-Type": "application/json", 94 | "Authorization": "Bearer {}".format(self.user_token), 95 | } 96 | 97 | @property 98 | def _state(self): 99 | """ 100 | :return: State of all field variables 101 | :rtype: dict 102 | """ 103 | return json.dumps( 104 | { 105 | "user_token": self.user_token, 106 | "repository": self.repository, 107 | "base_url": self.base_url, 108 | } 109 | ) 110 | 111 | def _streaming_query( 112 | self, 113 | query_string, 114 | start=None, 115 | end=None, 116 | is_live=None, 117 | timezone_offset_minutes=None, 118 | arguments=None, 119 | raw_data=None, 120 | media_type="application/x-ndjson", 121 | **kwargs 122 | ): 123 | """ 124 | Method wrapped by streaming_query to perform a Humio streaming Query. 125 | 126 | :return: An iterable that contains query results from stream as raw strings 127 | :rtype: Webstreamer 128 | """ 129 | 130 | if raw_data is None: 131 | raw_data = {} 132 | 133 | endpoint = "dataspaces/{}/query".format(self.repository) 134 | 135 | headers = self._default_user_headers 136 | headers["Accept"] = media_type 137 | headers.update(kwargs.pop("headers", {})) 138 | 139 | data = dict( 140 | (k, v) 141 | for k, v in [ 142 | ("queryString", query_string), 143 | ("start", start), 144 | ("end", end), 145 | ("isLive", is_live), 146 | ("timeZoneOffsetMinutes", timezone_offset_minutes), 147 | ("arguments", arguments), 148 | ] 149 | if v is not None 150 | ) 151 | 152 | data.update(raw_data) 153 | 154 | connection = self.webcaller.call_rest( 155 | "post", endpoint, data=json.dumps(data), headers=headers, stream=True, **kwargs 156 | ) 157 | 158 | return WebStreamer(connection) 159 | 160 | # Wrap method to be pythonic 161 | def streaming_query( 162 | self, 163 | query_string, 164 | start=None, 165 | end=None, 166 | is_live=None, 167 | timezone_offset_minutes=None, 168 | arguments=None, 169 | raw_data=None, 170 | **kwargs 171 | ): 172 | """ 173 | Humio Query type that opens up a streaming socket connection to Humio. 174 | This is the preferred way to do static queries with large result sizes. 175 | It can be used for live queries, but not that if data is not passed back from 176 | Humio for a while, the connection will be lost, resulting in an error. 177 | 178 | :param query_string: Humio query 179 | :type query_string: str 180 | :param start: Starting time of query 181 | :type start: Union[int, str], optional 182 | :param end: Ending time of query 183 | :type end: Union[int, str], optional 184 | :param is_live: Ending time of query 185 | :type is_live: bool, optional 186 | :param timezone_offset_minutes: Timezone offset in minutes 187 | :type timezone_offset_minutes: int, optional 188 | :param argument: Arguments specified in query 189 | :type argument: dict(string->string), optional 190 | :param raw_data: Additional arguments to add to POST body under other keys 191 | :type raw_data: dict(string->string), optional 192 | 193 | :return: A generator that returns query results as python objects 194 | :rtype: Generator 195 | """ 196 | 197 | media_type = "application/x-ndjson" 198 | encoding = "utf-8" 199 | 200 | res = self._streaming_query( 201 | query_string=query_string, 202 | start=start, 203 | end=end, 204 | is_live=is_live, 205 | timezone_offset_minutes=timezone_offset_minutes, 206 | arguments=arguments, 207 | media_type=media_type, 208 | raw_data=raw_data, 209 | **kwargs 210 | ) 211 | 212 | for event in res: 213 | yield json.loads(event.decode(encoding)) 214 | 215 | def create_queryjob( 216 | self, 217 | query_string, 218 | start=None, 219 | end=None, 220 | is_live=None, 221 | timezone_offset_minutes=None, 222 | arguments=None, 223 | raw_data=None, 224 | **kwargs 225 | ): 226 | """ 227 | Creates a queryjob on Humio, which executes asynchronously of the calling code. 228 | The returned QueryJob instance can be used to get the query results at a later time. 229 | Queryjobs are good to use for live queries, or static queries that return smaller 230 | amounts of data. 231 | 232 | :param query_string: Humio query 233 | :type query_string: str 234 | :param start: Starting time of query 235 | :type start: Union[int, str], optional 236 | :param end: Ending time of query 237 | :type end: Union[int, str], optional 238 | :param is_live: Ending time of query 239 | :type is_live: bool, optional 240 | :param is_live: Timezone offset in minutes 241 | :type is_live: int, optional 242 | :param argument: Arguments specified in query 243 | :type argument: dict(string->string), optional 244 | :param raw_data: Additional arguments to add to POST body under other keys 245 | :type raw_data: dict(string->string), optional 246 | 247 | :return: An instance that grants access to the created queryjob and associated results 248 | :rtype: QueryJob 249 | """ 250 | 251 | endpoint = "dataspaces/{}/queryjobs".format(self.repository) 252 | 253 | headers = self._default_user_headers 254 | headers.update(kwargs.pop("headers", {})) 255 | 256 | data = dict( 257 | (k, v) 258 | for k, v in [ 259 | ("queryString", query_string), 260 | ("start", start), 261 | ("end", end), 262 | ("isLive", is_live), 263 | ("timeZoneOffsetMinutes", timezone_offset_minutes), 264 | ("arguments", arguments), 265 | ] 266 | if v is not None 267 | ) 268 | 269 | if raw_data is not None: 270 | data.update(raw_data) 271 | 272 | query_id = self.webcaller.call_rest( 273 | "post", endpoint, data=json.dumps(data), headers=headers, **kwargs 274 | ).json()['id'] 275 | 276 | if is_live: 277 | return LiveQueryJob(query_id, self.base_url, self.repository, self.user_token) 278 | else: 279 | return StaticQueryJob(query_id, self.base_url, self.repository, self.user_token) 280 | 281 | def _ingest_json_data(self, json_elements=None, **kwargs): 282 | """ 283 | Ingest structured json data to repository. 284 | Structure of ingested data is discussed in: https://docs.humio.com/reference/api/ingest/#structured-data 285 | 286 | :param messages: A list of event strings. 287 | :type messages: list(string), optional 288 | :param parser: Name of parser to use on messages. 289 | :type parser: string, optional 290 | :param fields: Fields that should be added to events after parsing. 291 | :type fields: dict(string->string), optional 292 | :param tags: Tags to associate with the messages. 293 | :type tags: dict(string->string), optional 294 | 295 | :return: Response to web request as json string 296 | :rtype: str 297 | """ 298 | 299 | if json_elements is None: 300 | json_elements = [] 301 | 302 | headers = self._default_user_headers 303 | headers.update(kwargs.pop("headers", {})) 304 | 305 | endpoint = "dataspaces/{}/ingest".format(self.repository) 306 | 307 | return self.webcaller.call_rest( 308 | "post", endpoint, data=json.dumps(json_elements), headers=headers, **kwargs 309 | ) 310 | 311 | # Wrap method to be pythonic 312 | ingest_json_data = WebCaller.response_as_json(_ingest_json_data) 313 | 314 | def _ingest_messages( 315 | self, messages=None, parser=None, fields=None, tags=None, **kwargs 316 | ): 317 | """ 318 | Ingest unstructred messages to repository. 319 | Structure of ingested data is discussed in: https://docs.humio.com/reference/api/ingest/#parser 320 | 321 | :param messages: A list of event strings. 322 | :type messages: list(string), optional 323 | :param parser: Name of parser to use on messages. 324 | :type parser: string, optional 325 | :param fields: Fields that should be added to events after parsing. 326 | :type fields: dict(string->string), optional 327 | :param tags: Tags to associate with the messages. 328 | :type tags: dict(string->string), optional 329 | 330 | :return: Response to web request as json string 331 | :rtype: str 332 | """ 333 | if messages is None: 334 | messages = [] 335 | 336 | headers = self._default_user_headers 337 | headers.update(kwargs.pop("headers", {})) 338 | 339 | endpoint = "dataspaces/{}/ingest-messages".format(self.repository) 340 | 341 | obj = self._create_unstructured_data_object( 342 | messages, parser=parser, fields=fields, tags=tags 343 | ) 344 | 345 | return self.webcaller.call_rest( 346 | "post", endpoint, data=json.dumps([obj]), headers=headers, **kwargs 347 | ) 348 | 349 | # Wrap method to be pythonic 350 | ingest_messages = WebCaller.response_as_json(_ingest_messages) 351 | 352 | # status 353 | def _get_status(self, **kwargs): 354 | """ 355 | Gets status of Humio instance 356 | 357 | :return: Response to web request as json string 358 | :rtype: str 359 | """ 360 | endpoint = "status" 361 | return self.webcaller.call_rest("get", endpoint, **kwargs) 362 | 363 | # Wrap method to be pythonic 364 | get_status = WebCaller.response_as_json(_get_status) 365 | 366 | # user management 367 | def _get_users(self): 368 | """ 369 | Gets users registered to Humio instance 370 | 371 | :return: Response to web request as json string 372 | :rtype: str 373 | """ 374 | endpoint = "users" 375 | return self.webcaller.call_rest("get", endpoint, headers=self._default_user_headers) 376 | 377 | # Wrap method to be pythonic 378 | get_users = WebCaller.response_as_json(_get_users) 379 | 380 | def get_user_by_email(self, email): 381 | """ 382 | Get a user associated with Humio instance by email 383 | 384 | :param email: Email of queried user 385 | :type email: str 386 | 387 | :return: Response to web request as json string 388 | :rtype: str 389 | """ 390 | user_list = self.get_users() 391 | for user in user_list: 392 | if email == user["email"]: 393 | return user 394 | return None 395 | 396 | def _create_user(self, email, isRoot=False): 397 | """ 398 | Create user on Humio instance. Method is idempotent 399 | 400 | :param email: Email of user to create 401 | :type email: str 402 | :param isRoot: Indicates whether user should be root 403 | :type isRoot: bool, optional 404 | 405 | :return: Response to web request as json string 406 | :rtype: str 407 | """ 408 | 409 | endpoint = "users" 410 | 411 | data = {"email": email, "isRoot": isRoot} 412 | 413 | return self.webcaller.call_rest( 414 | "post", endpoint, data=json.dumps(data), headers=self._default_user_headers 415 | ) 416 | 417 | # Wrap method to be pythonic 418 | create_user = WebCaller.response_as_json(_create_user) 419 | 420 | def _delete_user_by_id(self, user_id): 421 | """ 422 | Delete user from Humio instance. 423 | 424 | :param user_id: Id of user to delete. 425 | :type user_id: string 426 | 427 | :return: Response to web request as json string 428 | :rtype: str 429 | """ 430 | 431 | link = "users/{}".format(user_id) 432 | 433 | return self.webcaller.call_rest("delete", link, headers=self._default_user_headers) 434 | 435 | # Wrap method to be pythonic 436 | delete_user_by_id = WebCaller.response_as_json(_delete_user_by_id) 437 | 438 | def delete_user_by_email(self, email): 439 | """ 440 | Delete user by email. 441 | 442 | :param email: Email of user to delete. 443 | :type email: string 444 | 445 | :return: Response to web request as json string 446 | :rtype: str 447 | """ 448 | for user in self.get_users(): 449 | if email == user["email"]: 450 | return self.delete_user_by_id(user["userID"]) 451 | return None 452 | 453 | # organizations 454 | def _list_organizations(self): 455 | """ 456 | List organizations. 457 | 458 | :return: Response to web request as json string 459 | :rtype: str 460 | """ 461 | 462 | headers = self._default_user_headers 463 | request = { 464 | "query": "query {organizations{id, name, description}}", 465 | "variables": None, 466 | } 467 | 468 | return self.webcaller.call_graphql(headers=headers, data=json.dumps(request)) 469 | 470 | # Wrap method to be pythonic 471 | def list_organizations(self): 472 | resp = self._list_organizations() 473 | return resp.json()["data"]["organizations"] 474 | 475 | def _create_organization(self, name, description): 476 | """ 477 | Create new organiztion. 478 | 479 | :param name: Name of organization. 480 | :type name: string 481 | :param description: Description of organization. 482 | :type description: string 483 | 484 | :return: Response to web request as json string 485 | :rtype: str 486 | """ 487 | 488 | headers = self._default_user_headers 489 | request = { 490 | "query": "mutation($name: String!, $description: String!){createOrganization(name: $name, description: $description){organization{id}}}", 491 | "variables": {"name": name, "description": description}, 492 | } 493 | return self.webcaller.call_graphql(headers=headers, data=json.dumps(request)) 494 | 495 | # Wrap method to be pythonic 496 | def create_organization(self, name, description): 497 | resp = self._create_organization(name, description) 498 | return resp.json()["data"] 499 | 500 | # files API 501 | def _upload_file(self, filepath): 502 | """ 503 | Upload file to repository 504 | 505 | :param filepath: Path to file. 506 | :type filepath: string 507 | 508 | :return: Response to web request 509 | :rtype: Response Object 510 | """ 511 | 512 | endpoint = "dataspaces/{}/files".format(self.repository) 513 | headers = {"Authorization": "Bearer {}".format(self.user_token)} # Not using default headers as files are sent 514 | with open(filepath, "rb") as f: 515 | return self.webcaller.call_rest("post", endpoint, files={"file": f}, headers=headers) 516 | 517 | # Wrap method to be pythonic 518 | # The uploaded files endpoint currently doesn't return JSON, thus this function doesn't attempt to cast to json. 519 | def upload_file(self, filepath): 520 | return self._upload_file(filepath) 521 | 522 | def _create_file(self, file_name): 523 | """ 524 | Create new file. 525 | 526 | :param file_name: Name of file 527 | :type file_name: string 528 | 529 | :return: Response to web request 530 | :rtype: Response Object 531 | """ 532 | 533 | headers = self._default_user_headers 534 | request = { 535 | "query": "mutation($fileName : String!, $repo : String!){newFile(fileName: $fileName, name: $repo){nameAndPath { name, path}}}", 536 | "variables": {"fileName": file_name, "repo": self.repository}, 537 | } 538 | return self.webcaller.call_graphql(headers=headers, data=json.dumps(request)) 539 | 540 | def create_file(self, file_name): 541 | """ 542 | Create new file. 543 | 544 | :param file_name: Name of file 545 | :type file_name: string 546 | 547 | :return: Response data to web request as json string 548 | :rtype: str 549 | """ 550 | 551 | resp = self._create_file(file_name) 552 | return resp.json()["data"] 553 | 554 | def _list_files(self): 555 | """ 556 | List uploaded files on repository 557 | 558 | :return: Response to web request 559 | :rtype: Response Object 560 | """ 561 | 562 | headers = self._default_user_headers 563 | request = { 564 | "query": "query {{searchDomain(name: {}){{files {{nameAndPath {{path, name}} }} }} }}".format( 565 | json.dumps(self.repository) 566 | ), 567 | "variables": None, 568 | } 569 | return self.webcaller.call_graphql(headers=headers, data=json.dumps(request)) 570 | 571 | def list_files(self): 572 | """ 573 | List uploaded files on repository 574 | 575 | :return: Response to web request as json string 576 | :rtype: str 577 | """ 578 | 579 | resp = self._list_files() 580 | return resp.json()["data"]["searchDomain"]["files"] 581 | 582 | def _get_file_content(self, file_name, offset, limit, filter_string=None): 583 | """ 584 | Get the contents of a file 585 | 586 | :param file_name: Name of file. 587 | :type name: string 588 | 589 | :param offset: Starting index to replace the old rows with the updated ones. 590 | :type offset: int 591 | 592 | :param limit: Used to find when to stop replacing rows, by adding the limit to the offset 593 | :type limit: int 594 | 595 | :param filter_string: Used to apply a filter string 596 | :type filter_string: string, optional 597 | 598 | :return: Response to web request 599 | :rtype: Response Object 600 | """ 601 | 602 | headers = self._default_user_headers 603 | 604 | if filter_string is not None: 605 | request = { 606 | "query": "query {{" 607 | "getFileContent(name: {}, fileName: \"{}\", offset: {}, limit: {}, filterString: \"{}\") {{ " 608 | "totalLinesCount, limit, offset, headers, lines}} " 609 | "}}".format(json.dumps(self.repository), file_name, offset, limit, filter_string), 610 | "variables": None, 611 | } 612 | else: 613 | request = { 614 | "query": "query {{" 615 | "getFileContent(name: {}, fileName: \"{}\", offset: {}, limit: {}) {{ " 616 | "totalLinesCount, limit, offset, headers, lines}} " 617 | "}}".format(json.dumps(self.repository), file_name, offset, limit), 618 | "variables": None, 619 | } 620 | 621 | return self.webcaller.call_graphql(headers=headers, data=json.dumps(request)) 622 | 623 | def get_file_content(self, filename, offset=0, limit=200, filter_string=None): 624 | """ 625 | Get the contents of a file 626 | 627 | :param file_name: Name of file. 628 | :type name: string 629 | 630 | :param offset: Starting index to replace the old rows with the updated ones. 631 | :type offset: int 632 | 633 | :param limit: Used to find when to stop replacing rows, by adding the limit to the offset 634 | :type limit: int 635 | 636 | :param filter_string: Used to apply a filter string 637 | :type filter_string: string, optional 638 | 639 | :return: Response to web request as json string 640 | :rtype: str 641 | """ 642 | 643 | resp = self._get_file_content(filename, offset=offset, limit=limit, filter_string=filter_string) 644 | return resp.json()["data"] 645 | 646 | def _get_file(self, file_name): 647 | """ 648 | Get specific file on repository 649 | 650 | :param file_name: Name of file to get. 651 | :type file_name: string 652 | 653 | :return: Response to web request as json string 654 | :rtype: Response Object 655 | """ 656 | endpoint = "dataspaces/{}/files/{}".format(self.repository, file_name) 657 | headers = {"Authorization": "Bearer {}".format(self.user_token)} # Not using default headers as files are sent 658 | return self.webcaller.call_rest("get", endpoint, headers=headers) 659 | 660 | def get_file(self, file_name, encoding=None): 661 | """ 662 | Get specific file on repository 663 | 664 | :param file_name: Name of file to get. 665 | :type file_name: string 666 | 667 | :return: Response to web request as json string 668 | :rtype: str 669 | """ 670 | resp = self._get_file(file_name) 671 | raw_data = resp.content 672 | if encoding is None: 673 | return raw_data 674 | else: 675 | return raw_data.decode("utf-8") 676 | 677 | def _delete_file(self, file_name): 678 | """ 679 | Delete an existing file. 680 | 681 | :param file_name: Name of file 682 | :type file_name: string 683 | 684 | :return: Response to web request 685 | :rtype: Response Object 686 | """ 687 | 688 | headers = self._default_user_headers 689 | request = { 690 | "query": "mutation($fileName : String!, $repo : String!){removeFile(fileName: $fileName, name: $repo){ __typename}}", 691 | "variables": {"fileName": file_name, "repo": self.repository}, 692 | } 693 | return self.webcaller.call_graphql(headers=headers, data=json.dumps(request)) 694 | 695 | def delete_file(self, file_name): 696 | """ 697 | Delete an existing file. 698 | 699 | :param file_name: Name of file 700 | :type file_name: string 701 | 702 | :return: Response to web request as json string 703 | :rtype: str 704 | """ 705 | 706 | resp = self._delete_file(file_name) 707 | return resp.json()["data"] 708 | 709 | def _update_file_contents(self, file_name, file_headers, changed_rows, column_changes=[], offset=0, limit=200): 710 | """ 711 | Add contents to a file 712 | 713 | :param file_name: Name of file 714 | :type file_name: string 715 | 716 | :param file_headers: Headers of the file 717 | :type file_headers: list 718 | 719 | :param changed_rows: Rows within the offset and limit to overwrite existing rows 720 | :type changed_rows: list 721 | 722 | :param column_changes: Column changes that will be applied to all rows in the file 723 | :type column_changes: list, optional 724 | 725 | :param offset: Starting index to replace the old rows with the updated ones. 726 | :type offset: int, optional 727 | 728 | :param limit: Used to find when to stop adding rows, by adding the limit to the offset 729 | :type limit: int, optional 730 | 731 | :return: Response data to web request 732 | :rtype: Response Object 733 | """ 734 | 735 | headers = self._default_user_headers 736 | request = { 737 | "query": "mutation ($fileName: String!, $name: String!, $changedRows: [[String!]!]!, $headers: [String!]!, $columnChanges: [ColumnChange!]!, $limit: Int, $offset: Int) {updateFile(limit: $limit, offset: $offset, fileName: $fileName, name: $name, changedRows: $changedRows, headers: $headers, columnChanges: $columnChanges) {offset, limit, totalLinesCount, headers, lines, nameAndPath {name, path } } }", 738 | "variables": {"name": self.repository, "fileName": file_name, "changedRows": changed_rows, 739 | "headers": file_headers, 740 | "columnChanges": column_changes, "offset": offset, "limit": limit} 741 | } 742 | 743 | return self.webcaller.call_graphql(headers=headers, data=json.dumps(request)) 744 | 745 | def add_file_contents(self, file_name, file_headers, changed_rows, column_changes=[], offset=0, limit=200): 746 | """ 747 | Add contents to a file 748 | 749 | :param file_name: Name of file 750 | :type file_name: string 751 | 752 | :param file_headers: Headers of the file 753 | :type file_headers: list 754 | 755 | :param changed_rows: Rows within the offset and limit to overwrite existing rows 756 | :type changed_rows: list 757 | 758 | :param column_changes: Column changes that will be applied to all rows in the file 759 | :type column_changes: list, optional 760 | 761 | :param offset: Starting index to replace the old rows with the updated ones. 762 | :type offset: int, optional 763 | 764 | :param limit: Used to determine when to stop replacing rows, by adding the limit to the offset 765 | :type limit: int, optional 766 | 767 | :return: Response data to web request as json string 768 | :rtype: str 769 | """ 770 | 771 | resp = self._update_file_contents(file_name, file_headers, changed_rows, column_changes, offset, limit) 772 | return resp.json()["data"] 773 | 774 | def remove_file_contents(self, file_name, offset=0, limit=200): 775 | """ 776 | Remove contents of a file 777 | 778 | :param file_name: Name of file 779 | :type file_name: string 780 | 781 | :param offset: Starting index to replace the old rows with the updated ones. 782 | :type offset: int, optional 783 | 784 | :param limit: Used to find when to stop replacing rows, by adding the limit to the offset 785 | :type limit: int, optional 786 | 787 | :return: Response data to web request as json string 788 | :rtype: str 789 | """ 790 | 791 | resp = self._update_file_contents(file_name, file_headers=[], offset=offset, limit=limit, changed_rows=[], 792 | column_changes=[]) 793 | return resp.json()["data"] 794 | 795 | def _create_saved_query(self, query_name, query_string): 796 | """ 797 | Create new saved query in the current repository. 798 | 799 | :param query_name: Name of saved query 800 | :type query_name: string 801 | 802 | :param query_string: Saved query content 803 | :type query_string: string 804 | 805 | :return: Response to web request 806 | :rtype: Response Object 807 | """ 808 | 809 | headers = self._default_user_headers 810 | request = { 811 | "query": "mutation($input : CreateSavedQueryInput!){createSavedQuery(input: $input){savedQuery{id, name}}}", 812 | "variables": {"input": {"name": query_name, "viewName": self.repository, "queryString": query_string}}, 813 | } 814 | return self.webcaller.call_graphql(headers=headers, data=json.dumps(request)) 815 | 816 | def create_saved_query(self, query_name, query_string): 817 | """ 818 | Create new saved query in the current repository. 819 | 820 | :param query_name: Name of saved query 821 | :type query_name: string 822 | 823 | :param query_string: Saved query content 824 | :type query_string: string 825 | 826 | :return: Response data to web request as json string 827 | :rtype: str 828 | """ 829 | 830 | resp = self._create_saved_query(query_name, query_string) 831 | return resp.json()["data"] 832 | 833 | def _list_saved_queries(self): 834 | """ 835 | List saved queries on repository 836 | 837 | :return: Response to web request 838 | :rtype: Response Object 839 | """ 840 | 841 | headers = self._default_user_headers 842 | request = { 843 | "query": "query {{repository(name: {}){{savedQueries {{ id, name, displayName, query {{queryString}} }} }} }}".format( 844 | json.dumps(self.repository) 845 | ), 846 | "variables": None, 847 | } 848 | return self.webcaller.call_graphql(headers=headers, data=json.dumps(request)) 849 | 850 | def list_saved_queries(self): 851 | """ 852 | List saved queries on repository 853 | 854 | :return: Response to web request as json string 855 | :rtype: str 856 | """ 857 | 858 | resp = self._list_saved_queries() 859 | return resp.json()["data"]["repository"]["savedQueries"] 860 | 861 | def _update_saved_query(self, query_id, updated_query_name, updated_query_string): 862 | """ 863 | Update the saved query with the given id in the current repository. 864 | 865 | :param query_id: ID of saved query to update 866 | :type query_id: string 867 | 868 | :param updated_query_name: Updated name of saved query 869 | :type updated_query_name: string 870 | 871 | :param updated_query_string: Updated content of the saved query 872 | :type updated_query_string: string 873 | 874 | :return: Response to web request 875 | :rtype: Response Object 876 | """ 877 | 878 | headers = self._default_user_headers 879 | request = { 880 | "query": "mutation($input : UpdateSavedQueryInput!){updateSavedQuery(input: $input){savedQuery{id, name}}}", 881 | "variables": {"input": {"id": query_id, "name": updated_query_name, "viewName": self.repository, "queryString": updated_query_string}}, 882 | } 883 | return self.webcaller.call_graphql(headers=headers, data=json.dumps(request)) 884 | 885 | def update_saved_query(self, query_id, updated_query_name, updated_query_string): 886 | """ 887 | Update the saved query with the given id in the current repository. 888 | 889 | :param query_id: ID of saved query to update 890 | :type query_id: string 891 | 892 | :param updated_query_name: Updated name of saved query 893 | :type updated_query_name: string 894 | 895 | :param updated_query_string: Updated content of the saved query 896 | :type updated_query_string: string 897 | 898 | :return: Response data to web request as json string 899 | :rtype: str 900 | """ 901 | 902 | resp = self._update_saved_query(query_id, updated_query_name, updated_query_string) 903 | return resp.json()["data"] 904 | 905 | def _delete_saved_query(self, query_id): 906 | """ 907 | Delete the saved query with the given id from the current repository. 908 | 909 | :param query_id: ID of saved query to update 910 | :type query_id: string 911 | 912 | :return: Response to web request 913 | :rtype: Response Object 914 | """ 915 | 916 | headers = self._default_user_headers 917 | request = { 918 | "query": "mutation($input : DeleteSavedQueryInput!){deleteSavedQuery(input: $input){savedQuery{id, name}}}", 919 | "variables": {"input": {"id": query_id, "viewName": self.repository}}, 920 | } 921 | return self.webcaller.call_graphql(headers=headers, data=json.dumps(request)) 922 | 923 | def delete_saved_query(self, query_id): 924 | """ 925 | Delete the saved query with the given id from the current repository. 926 | 927 | :param query_id: ID of saved query to update 928 | :type query_id: string 929 | 930 | :return: Response data to web request as json string 931 | :rtype: str 932 | """ 933 | 934 | resp = self._delete_saved_query(query_id) 935 | return resp.json()["data"] 936 | 937 | class HumioIngestClient(BaseHumioClient): 938 | """ 939 | A Humio client that is used exclusivly for ingesting data 940 | """ 941 | 942 | def __init__( 943 | self, 944 | ingest_token, 945 | base_url="http://localhost:3000", 946 | ): 947 | """ 948 | :param ingest_token: Ingest token to access ingest. 949 | :type ingest_token: string 950 | :param base_url: Url of Humio instance. 951 | :type base_url: string 952 | """ 953 | super().__init__(base_url) 954 | self.ingest_token = ingest_token 955 | self.webcaller = WebCaller(self.base_url) 956 | 957 | @property 958 | def _default_ingest_headers(self): 959 | """ 960 | :return: Default headers used for web requests 961 | :rtype: dict 962 | """ 963 | 964 | return { 965 | "Content-Type": "application/json", 966 | "Authorization": "Bearer {}".format(self.ingest_token), 967 | } 968 | 969 | @property 970 | def _state(self): 971 | """ 972 | :return: State of all field variables 973 | :rtype: dict 974 | """ 975 | 976 | return json.dumps( 977 | { 978 | "base_url": self.base_url, 979 | "ingest_token": self.ingest_token, 980 | } 981 | ) 982 | 983 | def _ingest_json_data(self, json_elements=None, **kwargs): 984 | """ 985 | Ingest structured json data to repository. 986 | Structure of ingested data is discussed in: https://docs.humio.com/reference/api/ingest/#structured-data 987 | 988 | :param json_elements: Structured data that can be parsed to a json string. 989 | :type json_elements: str 990 | 991 | :return: Response to web request as json string 992 | :rtype: str 993 | """ 994 | 995 | if json_elements is None: 996 | json_elements = [] 997 | 998 | headers = self._default_ingest_headers 999 | headers.update(kwargs.pop("headers", {})) 1000 | 1001 | endpoint = "ingest/humio-structured" 1002 | 1003 | return self.webcaller.call_rest( 1004 | "post", endpoint, data=json.dumps(json_elements), headers=headers, **kwargs 1005 | ) 1006 | 1007 | # Wrap method to be pythonic 1008 | ingest_json_data = WebCaller.response_as_json(_ingest_json_data) 1009 | 1010 | def _ingest_messages( 1011 | self, messages=None, parser=None, fields=None, tags=None, **kwargs 1012 | ): 1013 | """ 1014 | Ingest unstructred messages to repository. 1015 | Structure of ingested data is discussed in: https://docs.humio.com/reference/api/ingest/#parser 1016 | 1017 | :param messages: A list of event strings. 1018 | :type messages: list(string), optional 1019 | :param parser: Name of parser to use on messages. 1020 | :type parser: string, optional 1021 | :param fields: Fields that should be added to events after parsing. 1022 | :type fields: dict(string->string), optional 1023 | :param tags: Tags to associate with the messages. 1024 | :type tags: dict(string->string), optional 1025 | 1026 | :return: Response to web request as json string 1027 | :rtype: str 1028 | """ 1029 | 1030 | if messages is None: 1031 | messages = [] 1032 | 1033 | headers = self._default_ingest_headers 1034 | headers.update(kwargs.pop("headers", {})) 1035 | 1036 | endpoint = "ingest/humio-unstructured" 1037 | 1038 | obj = self._create_unstructured_data_object( 1039 | messages, parser=parser, fields=fields, tags=tags 1040 | ) 1041 | 1042 | return self.webcaller.call_rest("post", endpoint, data=json.dumps([obj]), headers=headers) 1043 | 1044 | # Wrap method to be pythonic 1045 | ingest_messages = WebCaller.response_as_json(_ingest_messages) 1046 | --------------------------------------------------------------------------------