├── CONTRIBUTING.md
├── LICENSE
├── README.md
├── __init__.py
├── common
├── __init__.py
├── api_utility.py
├── api_utility_test.py
├── chronicle_auth.py
├── chronicle_auth_test.py
├── commands_utility.py
├── commands_utility_test.py
├── constants
│ ├── __init__.py
│ ├── http_method.py
│ ├── key_constants.py
│ └── status.py
├── exception_handler.py
├── file_utility.py
├── options.py
├── templates.py
└── uri.py
├── feeds
├── __init__.py
├── commands
│ ├── __init__.py
│ ├── create.py
│ ├── create_test.py
│ ├── delete.py
│ ├── delete_test.py
│ ├── disable.py
│ ├── disable_test.py
│ ├── enable.py
│ ├── enable_test.py
│ ├── get.py
│ ├── get_test.py
│ ├── list.py
│ ├── list_test.py
│ ├── update.py
│ └── update_test.py
├── constants
│ ├── __init__.py
│ └── schema.py
├── feed_schema_utility.py
├── feed_schema_utility_test.py
├── feed_templates.py
├── feed_utility.py
├── feed_utility_test.py
├── feeds.py
├── feeds_test.py
└── tests
│ ├── __init__.py
│ └── fixtures.py
├── forwarders
├── __init__.py
├── collectors
│ ├── __init__.py
│ ├── collector_utility.py
│ ├── collector_utility_test.py
│ ├── collectors.py
│ ├── collectors_test.py
│ ├── commands
│ │ ├── __init__.py
│ │ ├── create.py
│ │ ├── create_test.py
│ │ ├── delete.py
│ │ ├── delete_test.py
│ │ ├── get.py
│ │ ├── get_test.py
│ │ ├── list.py
│ │ ├── list_test.py
│ │ ├── update.py
│ │ └── update_test.py
│ ├── constants
│ │ ├── __init__.py
│ │ └── schema.py
│ └── tests
│ │ ├── __init__.py
│ │ └── fixtures.py
├── commands
│ ├── __init__.py
│ ├── create.py
│ ├── create_test.py
│ ├── delete.py
│ ├── delete_test.py
│ ├── generate_files.py
│ ├── generate_files_test.py
│ ├── get.py
│ ├── get_test.py
│ ├── list.py
│ ├── list_test.py
│ ├── update.py
│ └── update_test.py
├── constants
│ ├── __init__.py
│ └── schema.py
├── forwarder_templates.py
├── forwarder_utility.py
├── forwarder_utility_test.py
├── forwarders.py
├── forwarders_test.py
├── schema_utility.py
├── schema_utility_test.py
├── schemas
│ ├── __init__.py
│ ├── collector_schema.json
│ └── forwarder_schema.json
└── tests
│ ├── __init__.py
│ └── fixtures.py
├── main.py
├── main_test.py
├── mock_test_utility.py
├── parsers
├── __init__.py
├── commands
│ ├── __init__.py
│ ├── activate_parser.py
│ ├── activate_parser_test.py
│ ├── archive.py
│ ├── archive_test.py
│ ├── classify_log_type.py
│ ├── classify_log_type_test.py
│ ├── deactivate_parser.py
│ ├── deactivate_parser_test.py
│ ├── delete_extension.py
│ ├── delete_extension_test.py
│ ├── delete_parser.py
│ ├── delete_parser_test.py
│ ├── download.py
│ ├── download_test.py
│ ├── generate.py
│ ├── generate_test.py
│ ├── get_extension.py
│ ├── get_extension_test.py
│ ├── get_parser.py
│ ├── get_parser_test.py
│ ├── get_validation_report.py
│ ├── get_validation_report_test.py
│ ├── history.py
│ ├── history_test.py
│ ├── list.py
│ ├── list_errors.py
│ ├── list_errors_test.py
│ ├── list_extensions.py
│ ├── list_extensions_test.py
│ ├── list_parsers.py
│ ├── list_parsers_test.py
│ ├── list_test.py
│ ├── run.py
│ ├── run_parser.py
│ ├── run_parser_test.py
│ ├── run_test.py
│ ├── status.py
│ ├── status_test.py
│ ├── submit.py
│ ├── submit_extension.py
│ ├── submit_extension_test.py
│ ├── submit_parser.py
│ ├── submit_parser_test.py
│ └── submit_test.py
├── constants
│ ├── __init__.py
│ ├── key_constants.py
│ └── path_constants.py
├── parser_templates.py
├── parser_utility.py
├── parser_utility_test.py
├── parsers.py
├── parsers_test.py
├── tests
│ ├── __init__.py
│ └── fixtures.py
├── url.py
└── url_test.py
├── requirements.txt
├── setup.py
└── tools
├── __init__.py
├── bigquery.py
├── bigquery_templates.py
├── bigquery_test.py
├── commands
├── __init__.py
├── provide_access.py
└── provide_access_test.py
├── constants
├── __init__.py
└── key_constants.py
├── tests
├── __init__.py
└── fixtures.py
├── url.py
└── url_test.py
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | # How to Contribute
2 |
3 | We'd love to accept your patches and contributions to this project. There are
4 | just a few small guidelines you need to follow.
5 |
6 | ## Contributor License Agreement
7 |
8 | Contributions to this project must be accompanied by a Contributor License
9 | Agreement. You (or your employer) retain the copyright to your contribution;
10 | this simply gives us permission to use and redistribute your contributions as
11 | part of the project. Head over to to see
12 | your current agreements on file or to sign a new one.
13 |
14 | You generally only need to submit a CLA once, so if you've already submitted one
15 | (even if it was for a different project), you probably don't need to do it
16 | again.
17 |
18 | ## Code Reviews
19 |
20 | All submissions, including submissions by project members, require review. We
21 | use GitHub pull requests for this purpose. Consult
22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
23 | information on using pull requests.
24 |
25 | ## Community Guidelines
26 |
27 | This project follows [Google's Open Source Community
28 | Guidelines](https://opensource.google/conduct/).
29 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Google Security Operations CLI
2 |
3 | Command line tool to interact with Google Security Operations' APIs.
4 |
5 | Google Security Operations CLI allows customers to manage various operations that can be
6 | performed on Google Security Operations. This script provides a command line tool to interact
7 | with Feed, Parser, Forwarder and BigQuery APIs. It will gradually expand to
8 | cover other APIs.
9 |
10 | ## Setup
11 |
12 | Follow these instructions: https://cloud.google.com/python/setup
13 |
14 | You may skip installing the Cloud Client Libraries and the Cloud SDK, they are
15 | unnecessary for interacting with Google Security Operations.
16 |
17 | After creating and activating the virtual environment `venv`, clone the repository using following command:
18 |
19 | ```shell
20 | git clone https://github.com/chronicle/cli.git
21 | ```
22 |
23 | After cloning, switch directory to `cli` and install Python library dependencies by running this command:
24 |
25 | ```shell
26 | cd cli
27 | pip install -r requirements.txt
28 | ```
29 |
30 | It is assumed that you're using Python 3.7 or above.
31 |
32 | ### Setting up a Python development environment
33 |
34 | https://cloud.google.com/python/docs/setup
35 |
36 | Go to root directory and execute following command:\
37 | ```shell
38 | python3 -m pip install --editable .
39 | ```
40 |
41 | ## Credentials
42 |
43 | Running the samples requires a JSON credentials file. By default, all the
44 | samples try to use the file `chronicle_credentials.json` from inside a hidden
45 | directory `.chronicle_cli` in the user's home directory. If this file is not
46 | found, you need to specify it explicitly by adding the following argument to the
47 | sample's command-line:
48 |
49 | ```shell
50 | --credential_file
51 | ```
52 |
53 | ## Run the chronicle_cli
54 |
55 | ```shell
56 | chronicle_cli --help
57 | ```
58 |
59 | ## Unit test case execution
60 |
61 | Execute the following command from root directory:\
62 | ```shell
63 | python3 -m pytest --cov=./ --cov-report term-missing -vv
64 | ```
65 |
66 |
67 | ## Documentation
68 |
69 | https://cloud.google.com/chronicle/docs/preview/cli-user-guide/cli-user-guide
70 |
--------------------------------------------------------------------------------
/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 |
--------------------------------------------------------------------------------
/common/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 |
--------------------------------------------------------------------------------
/common/api_utility.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Utility functions."""
16 |
17 | import json
18 | from typing import Any, AnyStr, Dict, Optional
19 |
20 | import click
21 |
22 | from common import templates
23 |
24 |
25 | def check_content_type(api_response: AnyStr) -> Any:
26 | """Return JSON based content for the response data.
27 |
28 | Args:
29 | api_response (AnyStr): API response
30 |
31 | Returns:
32 | JSON: Response data.
33 |
34 | Raises:
35 | TypeError: If response data is not JSON.
36 | """
37 | try:
38 | return json.loads(api_response)
39 | except json.JSONDecodeError:
40 | raise TypeError("URL is not reachable.") from None
41 |
42 |
43 | def print_request_details(url: AnyStr, method: AnyStr,
44 | request_body: Optional[Dict[str, Any]],
45 | response_body: Dict[str, Any]) -> None:
46 | """Prints HTTP request details to the console.
47 |
48 | Args:
49 | url (AnyStr): Request url
50 | method (AnyStr): Request method
51 | request_body (Optional[Dict[str,Any]]): Request body
52 | response_body (Dict[str,Any]): Response body
53 | """
54 | click.echo(
55 | templates.request_details_template.substitute(
56 | request_url=url,
57 | method=method,
58 | request_body=request_body,
59 | response_body=response_body,
60 | ))
61 |
--------------------------------------------------------------------------------
/common/api_utility_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Unit tests for api_utility.py."""
16 |
17 | from typing import Any
18 | import pytest
19 |
20 | from common import api_utility
21 |
22 |
23 | def test_content_type_is_json() -> None:
24 | """Test that the API response is JSON."""
25 | json_api_response = api_utility.check_content_type('{"key": "value"}')
26 | assert json_api_response == {'key': 'value'}
27 |
28 |
29 | def test_content_type_is_not_json() -> None:
30 | """Test that the API response is not JSON."""
31 | with pytest.raises(TypeError, match='URL is not reachable.'):
32 | api_utility.check_content_type('{"key": "value"')
33 |
34 |
35 | def test_print_request_details(capfd: Any) -> None:
36 | """Test printing request details."""
37 | api_utility.print_request_details('test.com', 'GET',
38 | {'header': 'test header'},
39 | {'body': 'test response'})
40 | console_output, _ = capfd.readouterr()
41 | assert console_output == """==========================================
42 | ========== HTTP Request Details ==========
43 | ==========================================
44 | Request:
45 | URL: test.com
46 | Method: GET
47 | Body: {'header': 'test header'}
48 | Response:
49 | Body: {'body': 'test response'}
50 |
51 | """
52 |
--------------------------------------------------------------------------------
/common/chronicle_auth.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Helper functions to access Chronicle APIs using OAuth 2.0."""
16 |
17 | import os
18 | import pathlib
19 | from typing import AnyStr, Any
20 |
21 | from google.auth.transport import requests
22 | from google.oauth2 import service_account
23 |
24 | CHRONICLE_CLI_ROOT_DIR = os.path.join(
25 | str(pathlib.Path.home()), ".chronicle_cli")
26 | default_cred_file_path = os.path.join(CHRONICLE_CLI_ROOT_DIR,
27 | "chronicle_credentials.json")
28 | AUTHORIZATION_SCOPES = ["https://www.googleapis.com/auth/chronicle-backstory"]
29 | DATAPLANE_AUTHORIZATION_SCOPES = [
30 | "https://www.googleapis.com/auth/cloud-platform"
31 | ]
32 |
33 |
34 | def initialize_http_session(credential_file_path: AnyStr) -> Any:
35 | """Initializes an authorized HTTP session, based on the given credential.
36 |
37 | Args:
38 | credential_file_path: Absolute or relative path to a JSON file containing
39 | private OAuth 2.0 credentials of a Google Cloud Platform service account.
40 | Default path is ".chronicle_credentials.json" in the .chronicle_cli
41 | directory inside user's home directory.
42 |
43 | Returns:
44 | HTTP session object to send authorized requests and receive responses.
45 | """
46 | credentials = service_account.Credentials.from_service_account_file(
47 | filename=os.path.abspath(credential_file_path or default_cred_file_path),
48 | scopes=AUTHORIZATION_SCOPES)
49 | return requests.AuthorizedSession(credentials)
50 |
51 |
52 | def initialize_dataplane_http_session(credential_file_path: AnyStr) -> Any:
53 | """Initalizes an authorized HTTP session for Dataplane APIs, based on the given credential.
54 |
55 | Args:
56 | credential_file_path: Absolute or relative path to a JSON file containing
57 | private OAuth 2.0 credentials of a Google Cloud Platform service account.
58 | Default path is ".chronicle_credentials.json" in the .chronicle_cli
59 | directory inside user's home directory.
60 |
61 | Returns:
62 | HTTP session object to send authorized requests and receive responses.
63 | """
64 | credentials = service_account.Credentials.from_service_account_file(
65 | filename=os.path.abspath(credential_file_path or default_cred_file_path),
66 | scopes=DATAPLANE_AUTHORIZATION_SCOPES)
67 | return requests.AuthorizedSession(credentials)
68 |
--------------------------------------------------------------------------------
/common/chronicle_auth_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Unit tests for chronicle_auth.py."""
16 |
17 | from unittest import mock
18 |
19 | from google.oauth2 import service_account
20 |
21 | from common import chronicle_auth
22 | from feeds.tests.fixtures import create_service_account_file
23 | from feeds.tests.fixtures import TEMP_SERVICE_ACCOUNT_FILE
24 |
25 |
26 | @mock.patch.object(service_account.Credentials, "from_service_account_file")
27 | def test_initialize_http_session(mock_from_service_account_file):
28 | """Test to check if http session is initialize or not.
29 |
30 | Args:
31 | mock_from_service_account_file (mock.MagicMock): Mock object
32 | """
33 | create_service_account_file()
34 | chronicle_auth.initialize_http_session("")
35 | mock_from_service_account_file.assert_called_once_with(
36 | filename=str(chronicle_auth.default_cred_file_path),
37 | scopes=chronicle_auth.AUTHORIZATION_SCOPES)
38 |
39 |
40 | @mock.patch.object(service_account.Credentials, "from_service_account_file")
41 | def test_initialize_http_session_with_custom_json_credentials(
42 | mock_from_service_account_file):
43 | """Test to check if http session is initialize with custom credentials.
44 |
45 | Args:
46 | mock_from_service_account_file (mock.MagicMock): Mock object
47 | """
48 | create_service_account_file()
49 | chronicle_auth.initialize_http_session(TEMP_SERVICE_ACCOUNT_FILE)
50 | mock_from_service_account_file.assert_called_once_with(
51 | filename=TEMP_SERVICE_ACCOUNT_FILE,
52 | scopes=chronicle_auth.AUTHORIZATION_SCOPES)
53 |
54 |
55 | @mock.patch.object(service_account.Credentials, "from_service_account_file")
56 | def test_initialize_chronicle_http_session(mock_from_service_account_file):
57 | """Test to check if http session is initialize or not.
58 |
59 | Args:
60 | mock_from_service_account_file (mock.MagicMock): Mock object
61 | """
62 | create_service_account_file()
63 | chronicle_auth.initialize_dataplane_http_session("")
64 | mock_from_service_account_file.assert_called_once_with(
65 | filename=str(chronicle_auth.default_cred_file_path),
66 | scopes=chronicle_auth.DATAPLANE_AUTHORIZATION_SCOPES)
67 |
68 |
69 | @mock.patch.object(service_account.Credentials, "from_service_account_file")
70 | def test_initialize_chronicle_http_session_with_custom_json_credentials(
71 | mock_from_service_account_file):
72 | """Test to check if http session is initialize with custom credentials.
73 |
74 | Args:
75 | mock_from_service_account_file (mock.MagicMock): Mock object
76 | """
77 | create_service_account_file()
78 | chronicle_auth.initialize_dataplane_http_session(TEMP_SERVICE_ACCOUNT_FILE)
79 | mock_from_service_account_file.assert_called_once_with(
80 | filename=TEMP_SERVICE_ACCOUNT_FILE,
81 | scopes=chronicle_auth.DATAPLANE_AUTHORIZATION_SCOPES)
82 |
--------------------------------------------------------------------------------
/common/constants/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 |
--------------------------------------------------------------------------------
/common/constants/http_method.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """HTTP method constants to be used across the project."""
16 |
17 | # TODO(sathishbabu): replace this module with the inbuilt http.HTTPMethod
18 | # when python >= 3.11
19 |
20 |
21 | DELETE = 'DELETE'
22 | GET = 'GET'
23 | PATCH = 'PATCH'
24 | POST = 'POST'
25 |
--------------------------------------------------------------------------------
/common/constants/key_constants.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Common constants to be used across the project."""
16 | KEY_ERROR = "error"
17 | KEY_ERRORS = "errors"
18 | KEY_MESSAGE = "message"
19 | KEY_LOG_TYPE = "logType"
20 |
--------------------------------------------------------------------------------
/common/constants/status.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """HTTP status constants to be used across the project."""
16 | import http
17 |
18 | STATUS_OK = http.HTTPStatus.OK.value
19 | STATUS_BAD_REQUEST = http.HTTPStatus.BAD_REQUEST.value
20 | STATUS_NOT_FOUND = http.HTTPStatus.NOT_FOUND.value
21 |
--------------------------------------------------------------------------------
/common/exception_handler.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Exception Handling Utility."""
16 |
17 | import functools
18 | import click
19 |
20 |
21 | def catch_exception(func=None):
22 | """Decorator to handle exceptions.
23 |
24 | Args:
25 | func: Original unwrapped function
26 |
27 | Returns:
28 | Result of calling function or prints exception in case of error.
29 | """
30 | if not func:
31 | return functools.partial(catch_exception)
32 |
33 | @functools.wraps(func)
34 | def wrapper(*args, **kwargs):
35 | """Wrapper function.
36 |
37 | Args:
38 | *args: User defined arguments
39 | **kwargs: Keyword user defined arguments
40 |
41 | Returns:
42 | Output of calling the function
43 | """
44 | try:
45 | return func(*args, **kwargs)
46 | except KeyError as e:
47 | click.echo(f"Failed to find key {str(e)} in the response.")
48 | except Exception as e: # pylint: disable=broad-except
49 | click.echo(f"Failed with exception: {str(e)}")
50 |
51 | return wrapper
52 |
--------------------------------------------------------------------------------
/common/file_utility.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Utility for file related operations."""
16 | import csv
17 | import json
18 | import os
19 | from typing import Any, AnyStr, Dict, List
20 |
21 | FILE_FORMAT_CSV = "CSV"
22 | FILE_FORMAT_JSON = "JSON"
23 | FILE_FORMAT_TXT = "TXT"
24 |
25 |
26 | def read_file(file_path: str) -> bytes:
27 | """Read file and return its content.
28 |
29 | Args:
30 | file_path: Path of file
31 |
32 | Returns:
33 | File Content
34 | """
35 | with open(file_path, "rb") as file:
36 | return file.read()
37 |
38 |
39 | def remove_file(file_path: str) -> None:
40 | """Removes the file if the path exists.
41 |
42 | Args:
43 | file_path (str): Path of the file to be removed.
44 | """
45 | if os.path.exists(file_path):
46 | os.remove(file_path)
47 |
48 |
49 | def export_json(file_path: AnyStr, json_data: Dict[str, Any]) -> None:
50 | """Write JSON data into file.
51 |
52 | Args:
53 | file_path (AnyStr): Path of file to export output of command.
54 | json_data (Dict): JSON data.
55 | """
56 | with open(file_path, "w") as file:
57 | file.write(json.dumps(json_data, indent=2))
58 |
59 |
60 | def export_csv(export_path: AnyStr, column_headers: List[str],
61 | rows: List[List[str]]) -> None:
62 | """Writes list of rows into csv file.
63 |
64 | Args:
65 | export_path (AnyStr): Path of file to export output of list command.
66 | column_headers (List[str]): List of all column name.
67 | rows (List[List[str]]): Array with row values.
68 | """
69 | with open(export_path, "w") as file:
70 | file_writer = csv.writer(file, delimiter=",")
71 | file_writer.writerow(column_headers)
72 | file_writer.writerows(rows)
73 |
74 |
75 | def export_txt(file_path: AnyStr, data: str) -> None:
76 | """Write text data into file.
77 |
78 | Args:
79 | file_path (AnyStr): Path of file to export output of command.
80 | data (str): Text data.
81 | """
82 | with open(file_path, "w") as file:
83 | file.write(data)
84 |
--------------------------------------------------------------------------------
/common/options.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """CLI options."""
16 |
17 | import click
18 |
19 | from common import chronicle_auth
20 |
21 | REGION_LIST = [
22 | "ASIA-NORTHEAST1",
23 | "ASIA-SOUTH1",
24 | "ASIA-SOUTHEAST1",
25 | "AUSTRALIA-SOUTHEAST1",
26 | "EUROPE",
27 | "EUROPE-WEST2",
28 | "EUROPE-WEST3",
29 | "EUROPE-WEST6",
30 | "ME-CENTRAL2",
31 | "ME-WEST1",
32 | "NORTHAMERICA-NORTHEAST2",
33 | "US",
34 | ]
35 |
36 | verbose_option = click.option(
37 | "--verbose", is_flag=True, help="Prints verbose output to the console."
38 | )
39 |
40 | credential_file_option = click.option(
41 | "-c",
42 | "--credential_file",
43 | help=(
44 | "Path of Service Account JSON. Default:"
45 | f" {chronicle_auth.default_cred_file_path}"
46 | ),
47 | )
48 |
49 | region_option = click.option(
50 | "--region",
51 | type=click.Choice(
52 | REGION_LIST,
53 | case_sensitive=False,
54 | ),
55 | default="US",
56 | help="Select region",
57 | )
58 |
59 | url_option = click.option("--url", help="Base URL to be used for API calls")
60 |
61 | env_option = click.option(
62 | "--env",
63 | type=click.Choice(["prod", "test"], case_sensitive=False),
64 | default="prod",
65 | help="""Optionally specify
66 | the environment for API calls""",
67 | )
68 |
69 | export_option = click.option(
70 | "--export", help="Export output to specified file path"
71 | )
72 |
73 | v2_option = click.option(
74 | "--v2", is_flag=True, help="Enable v2 commands."
75 | )
76 |
--------------------------------------------------------------------------------
/common/templates.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Common templates for printing output to console."""
16 | import string
17 |
18 | request_details_template = string.Template("""\
19 | ==========================================
20 | ========== HTTP Request Details ==========
21 | ==========================================
22 | Request:
23 | URL: ${request_url}
24 | Method: ${method}
25 | Body: ${request_body}
26 | Response:
27 | Body: ${response_body}
28 | """)
29 |
--------------------------------------------------------------------------------
/common/uri.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Helper functions to make Chronicle API URLs."""
16 |
17 | BASE_URL = "https://backstory.googleapis.com"
18 | CHRONICLE_TEST_API_URL = "https://test-backstory.sandbox.googleapis.com"
19 |
20 | DATAPLANE_BASE_URL = "https://{region}-chronicle.googleapis.com"
21 | DATAPLANE_TEST_URL = "https://test-chronicle.sandbox.googleapis.com"
22 |
23 |
24 | def get_base_url(region: str, custom_url: str, env: str = "prod") -> str:
25 | """Get base URL according to selected region.
26 |
27 | Args:
28 | region (str): Region (US, EUROPE, ASIA_SOUTHEAST1)
29 | custom_url (str): Base URL to be used for API calls
30 | env (str): Environment for API calls (prod, test)
31 |
32 | Returns:
33 | str: Base URL
34 | """
35 | if custom_url:
36 | return custom_url
37 | region = region.lower()
38 | if region != "us":
39 | return f"https://{region}-backstory.googleapis.com"
40 | if env == "test":
41 | return CHRONICLE_TEST_API_URL
42 | return BASE_URL
43 |
44 |
45 | def get_dataplane_base_url(region: str,
46 | custom_url: str,
47 | env: str = "prod") -> str:
48 | """Get base URL for DataPlane API according to selected region.
49 |
50 | Args:
51 | region (str): Region (US, EUROPE, ASIA_SOUTHEAST1, EUROPE_WEST2)
52 | custom_url (str): Base URL to be used for API calls
53 | env (str): Environment for API calls (prod, test)
54 |
55 | Returns:
56 | str: Base URL
57 | """
58 | if custom_url:
59 | return custom_url
60 | if env == "test":
61 | return DATAPLANE_TEST_URL
62 | return DATAPLANE_BASE_URL.format(region=region)
63 |
--------------------------------------------------------------------------------
/feeds/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 |
--------------------------------------------------------------------------------
/feeds/commands/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 |
--------------------------------------------------------------------------------
/feeds/commands/delete.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Delete feed."""
16 |
17 | from typing import AnyStr
18 |
19 | import click
20 |
21 | from common import api_utility
22 | from common import chronicle_auth
23 | from common import commands_utility
24 | from common import exception_handler
25 | from common import options
26 | from common.constants import key_constants
27 | from common.constants import status
28 | from feeds import feed_utility
29 |
30 |
31 | @click.command(help="Delete a feed")
32 | @options.url_option
33 | @options.region_option
34 | @options.verbose_option
35 | @options.credential_file_option
36 | @exception_handler.catch_exception()
37 | def delete(credential_file: AnyStr, verbose: bool, region: str,
38 | url: str) -> None:
39 | """Delete a Feed.
40 |
41 | Args:
42 | credential_file (AnyStr): Path of Service Account JSON.
43 | verbose (bool): Option for printing verbose output to console.
44 | region (str): Option for selecting regions. Available options - US, EUROPE,
45 | ASIA_SOUTHEAST1.
46 | url (str): Base URL to be used for API calls.
47 |
48 | Raises:
49 | OSError: Failed to read the given file, e.g. not found, no read access
50 | (https://docs.python.org/library/exceptions.html#os-exceptions).
51 | ValueError: Invalid file contents.
52 | KeyError: Required key is not present in dictionary.
53 | TypeError: If response data is not JSON.
54 | """
55 | url = commands_utility.lower_or_none(url)
56 | feed_id = click.prompt("Enter Feed ID", default="", show_default=False)
57 | if not feed_id:
58 | click.echo("Feed ID not provided. Please enter Feed ID.")
59 | return
60 |
61 | http_client = chronicle_auth.initialize_http_session(credential_file)
62 | full_url = f"{feed_utility.get_feed_url(region, url)}/{feed_id}"
63 | method = "DELETE"
64 | delete_feeds_response = http_client.request(method, full_url)
65 | status_code = delete_feeds_response.status_code
66 | response = api_utility.check_content_type(delete_feeds_response.text)
67 |
68 | if status_code == status.STATUS_OK:
69 | click.echo(f"\nFeed (ID: {feed_id}) deleted successfully.")
70 | elif status_code == status.STATUS_NOT_FOUND:
71 | click.echo("Invalid Feed ID. Please enter valid Feed ID.")
72 | elif status_code == status.STATUS_BAD_REQUEST:
73 | click.echo("Feed does not exist.")
74 | else:
75 | error_message = response[key_constants.KEY_ERROR][key_constants.KEY_MESSAGE]
76 | click.echo(f"\nError while deleting feed.\nResponse Code: {status_code}"
77 | f"\nError: {error_message}")
78 |
79 | if verbose:
80 | api_utility.print_request_details(full_url, method, None, response)
81 |
--------------------------------------------------------------------------------
/feeds/commands/delete_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Unit tests for delete.py ."""
16 |
17 | from unittest import mock
18 | from click.testing import CliRunner
19 | from feeds.commands.delete import delete
20 | from mock_test_utility import MockResponse
21 |
22 | runner = CliRunner()
23 |
24 |
25 | @mock.patch(
26 | "feeds.commands.delete.chronicle_auth.initialize_http_session"
27 | )
28 | @mock.patch(
29 | "feeds.commands.delete.click.prompt")
30 | def test_delete_200(input_patch: mock.MagicMock,
31 | mock_client: mock.MagicMock) -> None:
32 | """Test case to successfully delete feed.
33 |
34 | Args:
35 | input_patch (mock.MagicMock): Mock object
36 | mock_client (mock.MagicMock): Mock object
37 | """
38 | mock_client.return_value = mock.Mock()
39 | input_patch.return_value = "123"
40 | mock_client.return_value.request.side_effect = [
41 | MockResponse(status_code=200, text="""{}""")
42 | ]
43 |
44 | # Method Call
45 | result = runner.invoke(delete)
46 | assert "\nFeed (ID: 123) deleted successfully." in result.output
47 |
48 |
49 | @mock.patch(
50 | "feeds.commands.delete.chronicle_auth.initialize_http_session"
51 | )
52 | @mock.patch(
53 | "feeds.commands.delete.click.prompt")
54 | def test_delete_400(input_patch: mock.MagicMock,
55 | mock_client: mock.MagicMock) -> None:
56 | """Test case to check whether Feed ID does not exist.
57 |
58 | Args:
59 | input_patch (mock.MagicMock): Mock object
60 | mock_client (mock.MagicMock): Mock object
61 | """
62 | mock_client.return_value = mock.Mock()
63 | input_patch.return_value = "123"
64 | mock_client.return_value.request.side_effect = [
65 | MockResponse(status_code=400, text="""{}""")
66 | ]
67 |
68 | # Method Call
69 | result = runner.invoke(delete)
70 | assert "Feed does not exist." in result.output
71 |
72 |
73 | @mock.patch(
74 | "feeds.commands.delete.chronicle_auth.initialize_http_session"
75 | )
76 | @mock.patch(
77 | "feeds.commands.delete.click.prompt")
78 | def test_delete_404(input_patch: mock.MagicMock,
79 | mock_client: mock.MagicMock) -> None:
80 | """Test case to check for invalid feed ID.
81 |
82 | Args:
83 | input_patch (mock.MagicMock): Mock object
84 | mock_client (mock.MagicMock): Mock object
85 | """
86 | mock_client.return_value = mock.Mock()
87 | input_patch.return_value = "123"
88 | mock_client.return_value.request.side_effect = [
89 | MockResponse(status_code=404, text="""{}""")
90 | ]
91 |
92 | # Method Call
93 | result = runner.invoke(delete)
94 | assert "Invalid Feed ID. Please enter valid Feed ID." in result.output
95 |
96 |
97 | @mock.patch(
98 | "feeds.commands.delete.chronicle_auth.initialize_http_session"
99 | )
100 | @mock.patch(
101 | "feeds.commands.delete.click.prompt")
102 | def test_delete_credential_file(input_patch: mock.MagicMock,
103 | mock_client: mock.MagicMock) -> None:
104 | """Test case to check invalid credential path.
105 |
106 | Args:
107 | input_patch (mock.MagicMock): Mock object
108 | mock_client (mock.MagicMock): Mock object
109 | """
110 | mock_client.return_value = mock.Mock()
111 | input_patch.return_value = "123"
112 | mock_client.return_value.request.side_effect = OSError(
113 | "Credential Path not found.")
114 | expected_message = "Failed with exception: Credential Path not found."
115 |
116 | # Method Call
117 | result = runner.invoke(delete, ["--credential_file", "dummy.json"])
118 | assert expected_message in result.output
119 |
120 |
121 | @mock.patch(
122 | "feeds.commands.delete.click.prompt")
123 | def test_delete_feed_not_provided(input_patch: mock.MagicMock) -> None:
124 | """Test case to check if Feed ID not provided.
125 |
126 | Args:
127 | input_patch (mock.MagicMock): Mock object
128 | """
129 | input_patch.return_value = ""
130 |
131 | # Method Call
132 | result = runner.invoke(delete)
133 | assert "Feed ID not provided. Please enter Feed ID." in result.output
134 |
--------------------------------------------------------------------------------
/feeds/commands/disable.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Disable feed."""
16 |
17 | from typing import AnyStr
18 |
19 | import click
20 |
21 | from common import api_utility
22 | from common import commands_utility
23 | from common import exception_handler
24 | from common import options
25 | from common.constants import key_constants
26 | from common.constants import status
27 | from feeds import feed_schema_utility
28 | from feeds import feed_utility
29 |
30 |
31 | @click.command(help="Disable feed with a given feed id.")
32 | @options.url_option
33 | @options.region_option
34 | @options.verbose_option
35 | @options.credential_file_option
36 | @exception_handler.catch_exception()
37 | def disable(credential_file: AnyStr, verbose: bool, region: str,
38 | url: AnyStr) -> None:
39 | """Disable feed using Feed ID.
40 |
41 | Args:
42 | credential_file (str): Path of Service Account JSON.
43 | verbose (bool): Option for printing verbose output to console.
44 | region (str): Option for selecting regions. Available options - US, EUROPE,
45 | ASIA_SOUTHEAST1.
46 | url (str): Base URL to be used for API calls.
47 |
48 | Raises:
49 | OSError: Failed to read the given file, e.g. not found, no read access
50 | (https://docs.python.org/library/exceptions.html#os-exceptions).
51 | ValueError: Invalid file contents.
52 | KeyError: Required key is not present in dictionary.
53 | TypeError: If response data is not JSON.
54 | """
55 | url = commands_utility.lower_or_none(url)
56 | feed_schema = feed_schema_utility.FeedSchema(credential_file, region, url)
57 | feed_id = click.prompt("Enter Feed ID", default="", show_default=False)
58 | if not feed_id:
59 | click.echo("Feed ID not provided. Please enter Feed ID.")
60 | return
61 |
62 | full_url = f"{feed_utility.get_feed_url(region, url)}/{feed_id}:disable"
63 | method = "POST"
64 | disable_feed_response = feed_schema.client.request(method, full_url, {})
65 | response = api_utility.check_content_type(disable_feed_response.text)
66 |
67 | status_code = disable_feed_response.status_code
68 |
69 | if status_code == status.STATUS_OK:
70 | click.echo(f"Feed with ID: {feed_id} disabled successfully.")
71 | elif status_code == status.STATUS_NOT_FOUND:
72 | click.echo("Invalid Feed ID. Please enter valid Feed ID.")
73 | elif status_code == status.STATUS_BAD_REQUEST:
74 | click.echo("Feed does not exist.")
75 | else:
76 | click.echo(
77 | f"Error while disabling feed.\nResponse Code: {status_code}\nError: "
78 | f"{response[key_constants.KEY_ERROR][key_constants.KEY_MESSAGE]}")
79 |
80 | if verbose:
81 | api_utility.print_request_details(full_url, method, None, response)
82 |
--------------------------------------------------------------------------------
/feeds/commands/enable.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Enable feed."""
16 |
17 | from typing import AnyStr
18 |
19 | import click
20 |
21 | from common import api_utility
22 | from common import commands_utility
23 | from common import exception_handler
24 | from common import options
25 | from common.constants import key_constants
26 | from common.constants import status
27 | from feeds import feed_schema_utility
28 | from feeds import feed_utility
29 |
30 |
31 | @click.command(help="Enable feed with a given feed id.")
32 | @options.url_option
33 | @options.region_option
34 | @options.verbose_option
35 | @options.credential_file_option
36 | @exception_handler.catch_exception()
37 | def enable(credential_file: AnyStr, verbose: bool, region: str,
38 | url: AnyStr) -> None:
39 | """Enable feed using Feed ID.
40 |
41 | Args:
42 | credential_file (str): Path of Service Account JSON.
43 | verbose (bool): Option for printing verbose output to console.
44 | region (str): Option for selecting regions. Available options - US, EUROPE,
45 | ASIA_SOUTHEAST1.
46 | url (str): Base URL to be used for API calls.
47 |
48 | Raises:
49 | OSError: Failed to read the given file, e.g. not found, no read access
50 | (https://docs.python.org/library/exceptions.html#os-exceptions).
51 | ValueError: Invalid file contents.
52 | KeyError: Required key is not present in dictionary.
53 | TypeError: If response data is not JSON.
54 | """
55 | url = commands_utility.lower_or_none(url)
56 | feed_schema = feed_schema_utility.FeedSchema(credential_file, region, url)
57 | feed_id = click.prompt("Enter Feed ID", default="", show_default=False)
58 | if not feed_id:
59 | click.echo("Feed ID not provided. Please enter Feed ID.")
60 | return
61 |
62 | full_url = f"{feed_utility.get_feed_url(region, url)}/{feed_id}:enable"
63 | method = "POST"
64 | enable_feed_response = feed_schema.client.request(method, full_url, {})
65 | response = api_utility.check_content_type(enable_feed_response.text)
66 |
67 | status_code = enable_feed_response.status_code
68 |
69 | if status_code == status.STATUS_OK:
70 | click.echo(f"Feed with ID: {feed_id} enabled successfully.")
71 | elif status_code == status.STATUS_NOT_FOUND:
72 | click.echo("Invalid Feed ID. Please enter valid Feed ID.")
73 | elif status_code == status.STATUS_BAD_REQUEST:
74 | click.echo("Feed does not exist.")
75 | else:
76 | click.echo(
77 | f"Error while enabling feed.\nResponse Code: {status_code}\nError: "
78 | f"{response[key_constants.KEY_ERROR][key_constants.KEY_MESSAGE]}")
79 |
80 | if verbose:
81 | api_utility.print_request_details(full_url, method, None, response)
82 |
--------------------------------------------------------------------------------
/feeds/commands/get.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Get feed details."""
16 |
17 | from typing import AnyStr
18 |
19 | import click
20 |
21 | from common import api_utility
22 | from common import commands_utility
23 | from common import exception_handler
24 | from common import options
25 | from common.constants import key_constants
26 | from common.constants import status
27 | from feeds import feed_schema_utility
28 | from feeds import feed_templates
29 | from feeds import feed_utility
30 | from feeds.constants import schema
31 |
32 |
33 | @click.command(help="Get feed details using Feed ID")
34 | @options.url_option
35 | @options.region_option
36 | @options.verbose_option
37 | @options.credential_file_option
38 | @exception_handler.catch_exception()
39 | def get(credential_file: AnyStr, verbose: bool, region: str,
40 | url: AnyStr) -> None:
41 | """Get feed details using Feed ID.
42 |
43 | Args:
44 | credential_file (str): Path of Service Account JSON.
45 | verbose (bool): Option for printing verbose output to console.
46 | region (str): Option for selecting regions. Available options - US, EUROPE,
47 | ASIA_SOUTHEAST1.
48 | url (str): Base URL to be used for API calls.
49 |
50 | Raises:
51 | OSError: Failed to read the given file, e.g. not found, no read access
52 | (https://docs.python.org/library/exceptions.html#os-exceptions).
53 | ValueError: Invalid file contents.
54 | KeyError: Required key is not present in dictionary.
55 | TypeError: If response data is not JSON.
56 | """
57 | url = commands_utility.lower_or_none(url)
58 | feed_schema = feed_schema_utility.FeedSchema(credential_file, region, url)
59 | feed_id = click.prompt("Enter Feed ID", default="", show_default=False)
60 | if not feed_id:
61 | click.echo("Feed ID not provided. Please enter Feed ID.")
62 | return
63 |
64 | full_url = f"{feed_utility.get_feed_url(region, url)}/{feed_id}"
65 | method = "GET"
66 | get_feed_response = feed_schema.client.request(method, full_url)
67 | response = api_utility.check_content_type(get_feed_response.text)
68 |
69 | status_code = get_feed_response.status_code
70 |
71 | if status_code == status.STATUS_OK:
72 | detail_schema = feed_schema.get_detailed_schema(
73 | response[schema.KEY_DETAILS][schema.KEY_FEED_SOURCE_TYPE],
74 | response[schema.KEY_DETAILS][key_constants.KEY_LOG_TYPE])
75 | if detail_schema.error:
76 | click.echo(detail_schema.error)
77 | return
78 |
79 | flattened_response = commands_utility.flatten_dict(response)
80 | field_response = feed_utility.get_feed_details(
81 | flattened_response, detail_schema.log_type_schema)
82 | namespace = feed_utility.get_namespace(response.get(schema.KEY_DETAILS, {}))
83 | labels = feed_utility.get_labels(response.get(schema.KEY_DETAILS, {}))
84 | click.echo(
85 | feed_templates.feed_template.substitute(
86 | feed_id=feed_id,
87 | feed_display_name=feed_utility.get_feed_display_name(response),
88 | source_type=detail_schema.display_source_type,
89 | log_type=detail_schema.log_type_schema[schema.KEY_DISPLAY_NAME],
90 | feed_state=response[schema.KEY_FEED_STATE],
91 | feed_details=field_response,
92 | namespace=namespace,
93 | labels=labels))
94 |
95 | elif status_code == status.STATUS_NOT_FOUND:
96 | click.echo("Invalid Feed ID. Please enter valid Feed ID.")
97 | elif status_code == status.STATUS_BAD_REQUEST:
98 | click.echo("Feed does not exist.")
99 | else:
100 | click.echo(
101 | f"Error while fetching feed.\nResponse Code: {status_code}\nError: "
102 | f"{response[key_constants.KEY_ERROR][key_constants.KEY_MESSAGE]}")
103 |
104 | if verbose:
105 | api_utility.print_request_details(full_url, method, None, response)
106 |
--------------------------------------------------------------------------------
/feeds/constants/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 |
--------------------------------------------------------------------------------
/feeds/constants/schema.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Feed schema constants to be used across the project."""
16 |
17 | KEY_FEED_SOURCE_TYPE_SCHEMAS = "feedSourceTypeSchemas"
18 | KEY_FEED_SOURCE_TYPE = "feedSourceType"
19 | KEY_LOG_TYPE_SCHEMAS = "logTypeSchemas"
20 | KEY_LOG_TYPE_SCHEMA = "log_type_schema"
21 | KEY_LOG_TYPES = "logTypes"
22 | KEY_NAME = "name"
23 | KEY_DISPLAY_NAME = "displayName"
24 | KEY_DETAILED_FEED_SCHEMAS = "detailsFieldSchemas"
25 | KEY_ENUMFIELD_SCHEMAS = "enumFieldSchemas"
26 | KEY_FIELD_PATH = "fieldPath"
27 | KEY_STATUS = "status"
28 | KEY_FIELD_VALUE = "value"
29 | KEY_DETAILS = "details"
30 | KEY_FEED_STATE = "feedState"
31 | KEY_FEEDS = "feeds"
32 | KEY_DESCRIPTION = "description"
33 | KEY_DISPLAY_SOURCE_TYPE = "display_source_type"
34 | KEY_READ_ONLY = "readOnly"
35 | KEY_IS_REQUIRED = "isRequired"
36 | KEY_FIELD_TYPE = "type"
37 | KEY_DETAILS_FEED_SCHEMA_ALT = "detailsFieldSchemaAlternatives"
38 | KEY_DETAILS_FEED_SCHEMA_SET = "detailsFieldSchemaSets"
39 | ENUM_FIELD_TYPE = "ENUM"
40 | STR_SECRET_FIELD_TYPE = "STRING_SECRET"
41 | STR_MULTILINE_FIELD_TYPE = "STRING_MULTILINE"
42 | MAP_STR_FIELD_TYPE = "MAP_STRING_STRING"
43 | MULTILINE_SECRET_FIELD_TYPE = "STRING_MULTILINE_SECRET"
44 | KV_LIST_FIELD_TYPE = "KEY_VALUE_LIST"
45 | STR_LIST_FIELD_TYPE = "STRING_LIST"
46 | BOOL_FIELD_TYPE = "BOOL"
47 | JSON_CONTENT_TYPE = "application/json"
48 | KEY_CONTENT_TYPE = "content-type"
49 | KEY_DISPLAY_LOG_TYPE = "display_log_type"
50 | KEY_DETAILS_NAMESPACE = "details.namespace"
51 | KEY_DETAILS_LABELS = "details.labels"
52 | FEED_COLUMN_HEADER = [
53 | "ID", "Display Name", "Source type", "Log type", "State", "Feed Settings",
54 | "Namespace", "Labels"
55 | ]
56 |
--------------------------------------------------------------------------------
/feeds/feed_templates.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Templates for printing output to console."""
16 |
17 | import string
18 |
19 | feed_template = string.Template("""\
20 |
21 | Feed Details:
22 | ID: ${feed_id}\
23 | ${feed_display_name}
24 | Source type: ${source_type}
25 | Log type: ${log_type}
26 | State: ${feed_state}
27 | ${feed_details}\
28 | ${namespace}\
29 | ${labels}""")
30 |
31 | properties_template = string.Template("""\
32 | ====================================
33 | ========== Set Properties ==========
34 | ====================================""")
35 |
36 | log_type_template = string.Template("""\
37 | List of Log types:
38 |
39 | (i) How to select log type?
40 | - Press Up/b or Down/z keys to paginate.
41 | - To switch case-sensitivity, press '-i' and press enter. By default, search
42 | is case-sensitive.
43 | - To search for specific log type, press '/' key, enter text and press enter.
44 | - Note down the choice number for the log type that you want to select.
45 | - Press 'q' to quit and enter that choice number.
46 | - Press `h` for all the available options to navigate the list.
47 | =============================================================================
48 | """)
49 |
50 | log_type_template_win = string.Template("""\
51 | List of Log types:
52 |
53 | (i) How to select log type?
54 | - Press ENTER key (scrolls one line at a time) or SPACEBAR key (display next screen).
55 | - Note down the choice number for the log type that you want to select.
56 | - Press 'q' to quit and enter that choice number.
57 | =============================================================================
58 | """)
59 |
60 | input_parameters_template = string.Template("""\n\n\
61 | ======================================
62 | =========== Input Parameters =========
63 | ======================================
64 | (*) - Required fields.
65 | Password/secret inputs are hidden.""")
66 |
67 | retry_template = string.Template("""
68 | Looks like there was a failed feed create/update attempt with source type: ${source_type} and log type: ${log_type}.
69 | Would you like to retry?
70 | """)
71 |
--------------------------------------------------------------------------------
/feeds/feed_utility_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Unit tests for feed_utility.py."""
16 |
17 | from typing import Dict
18 | from feeds import feed_utility
19 | from feeds.tests.fixtures import * # pylint: disable=wildcard-import
20 |
21 |
22 | def test_defflatten_dict() -> None:
23 | """Test deflattening of dict."""
24 | input_dict = {"a.b": "c", "1.a": [1], "p.q.r": "s"}
25 | expected_output = {"a": {"b": "c"}, "1": {"a": [1]}, "p": {"q": {"r": "s"}}}
26 | assert feed_utility.deflatten_dict(input_dict) == expected_output
27 |
28 |
29 | def test_get_feed_details(get_flattened_response: Dict[str, str],
30 | get_detailed_schema: Dict[str, str]) -> None:
31 | """Test printing of key-value pair after correlation with schema.
32 |
33 | Args:
34 | get_flattened_response (dict): Test input data
35 | get_detailed_schema (dict): Test input data
36 | """
37 | expected_output = (" Feed Settings:\n API Hostname: abc.workday.com\n "
38 | "Tenant: ID\n")
39 | assert feed_utility.get_feed_details(
40 | get_flattened_response,
41 | get_detailed_schema.log_type_schema) == expected_output
42 |
43 |
44 | def test_snake_to_camel() -> None:
45 | """Test conversion of snakecase string to camelcase string."""
46 | assert feed_utility.snake_to_camel("feed_schema") == "feedSchema"
47 |
48 |
49 | def test_get_labels() -> None:
50 | """Test printing of key-value pair of labels field."""
51 | expected_output = (" Labels:\n k: v\n")
52 | assert feed_utility.get_labels({"labels": [{
53 | "key": "k",
54 | "value": "v"
55 | }]}) == expected_output
56 |
57 |
58 | def test_namespace() -> None:
59 | """Test printing of namespace field."""
60 | expected_output = (" Namespace: sample_namespace\n")
61 | assert feed_utility.get_namespace({"namespace": "sample_namespace"
62 | }) == expected_output
63 |
64 |
65 | def test_get_feed_display_name() -> None:
66 | """Test feed display name if exist in feed dictionary."""
67 | expected_output = "\n Display Name: Dummy feed display name"
68 | assert feed_utility.get_feed_display_name(
69 | {"displayName": "Dummy feed display name"}) == expected_output
70 |
71 |
72 | def test_get_feed_display_name_none() -> None:
73 | """Test feed display name if not exist in feed dictonary."""
74 | assert not feed_utility.get_feed_display_name({})
75 |
--------------------------------------------------------------------------------
/feeds/feeds.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Grouping Feed CLI commands."""
16 |
17 | import os
18 |
19 | import click
20 |
21 | from common import chronicle_auth
22 | from feeds.commands import create
23 | from feeds.commands import delete
24 | from feeds.commands import disable
25 | from feeds.commands import enable
26 | from feeds.commands import get
27 | from feeds.commands import list # pylint: disable=redefined-builtin
28 | from feeds.commands import update
29 |
30 |
31 | @click.group(name="feeds", help="Feed Management Workflows")
32 | def feeds() -> None:
33 | """Feeds group commands."""
34 | feed_dir = os.path.join(chronicle_auth.CHRONICLE_CLI_ROOT_DIR, "feeds")
35 | if not os.path.exists(feed_dir):
36 | os.mkdir(feed_dir)
37 |
38 |
39 | feeds.add_command(get.get)
40 | feeds.add_command(list.list_command)
41 | feeds.add_command(create.create)
42 | feeds.add_command(update.update)
43 | feeds.add_command(delete.delete)
44 | feeds.add_command(enable.enable)
45 | feeds.add_command(disable.disable)
46 |
--------------------------------------------------------------------------------
/feeds/feeds_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Unit tests for feeds.py."""
16 |
17 | from click.testing import CliRunner
18 |
19 | from feeds.feeds import feeds
20 |
21 |
22 | runner = CliRunner()
23 |
24 |
25 | def test_feeds() -> None:
26 | """Test case for feeds."""
27 | result = runner.invoke(feeds)
28 | expected_output = """Commands:
29 | create Create a feed
30 | delete Delete a feed
31 | disable Disable feed with a given feed id.
32 | enable Enable feed with a given feed id.
33 | get Get feed details using Feed ID
34 | list List all feeds
35 | update Update feed details using Feed ID"""
36 | assert expected_output in result.output
37 |
--------------------------------------------------------------------------------
/feeds/tests/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 |
--------------------------------------------------------------------------------
/forwarders/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 |
--------------------------------------------------------------------------------
/forwarders/collectors/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 |
--------------------------------------------------------------------------------
/forwarders/collectors/collector_utility.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Utility functions."""
16 | from typing import Any
17 |
18 | from common import api_utility
19 | from common import uri
20 | from common.constants import key_constants
21 | from common.constants import status
22 | from forwarders.constants import schema
23 |
24 | API_VERSION = "v2"
25 |
26 |
27 | def get_collector_url(region: str, custom_url: str, forwarder_id: str) -> str:
28 | """Gets collector URL according to selected region.
29 |
30 | Args:
31 | region (str): Region (US, EUROPE, ASIA_SOUTHEAST1).
32 | custom_url (str): Base URL to be used for API calls.
33 | forwarder_id (str): Forwarder uuid to which collector belongs.
34 |
35 | Returns:
36 | str: Collector URL.
37 | """
38 |
39 | return uri.get_base_url(
40 | region,
41 | custom_url) + f"/{API_VERSION}/forwarders/{forwarder_id}/collectors"
42 |
43 |
44 | def fetch_collectors(url: str, method: str, client: Any) -> Any:
45 | """Fetches list of collectors for respective forwarders.
46 |
47 | Args:
48 | url (str): Url to be used for API calls.
49 | method (str): Method to be used for API calls.
50 | client (Any): HTTP session object to send authorized requests and receive
51 | responses.
52 |
53 | Returns:
54 | Tuple(Dict, Dict): list of collectors response for forwarder.
55 | """
56 | # Store error message in dictionary if API returns bad status code
57 | # represent error on console in yaml format.
58 | collector_errors = {}
59 | collector_errors[schema.KEY_COLLECTORS] = {}
60 | list_collectors_response = {}
61 |
62 | list_collectors_response = client.request(method, url)
63 | status_code = list_collectors_response.status_code
64 | list_collectors_response = api_utility.check_content_type(
65 | list_collectors_response.text)
66 |
67 | if not list_collectors_response:
68 | collector_errors[schema.KEY_COLLECTORS][
69 | key_constants.KEY_MESSAGE] = "No collectors found for this forwarder."
70 |
71 | if status_code != status.STATUS_OK:
72 |
73 | collector_errors[schema.KEY_COLLECTORS][key_constants.KEY_ERROR] = {
74 | schema.KEY_RESPONSE_CODE:
75 | status_code,
76 | key_constants.KEY_MESSAGE:
77 | list_collectors_response[key_constants.KEY_ERROR]
78 | [key_constants.KEY_MESSAGE]
79 | }
80 |
81 | return list_collectors_response, collector_errors
82 |
--------------------------------------------------------------------------------
/forwarders/collectors/collectors.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Grouping Collectors CLI commands."""
16 |
17 | import os
18 |
19 | import click
20 |
21 | from common import chronicle_auth
22 | from forwarders.collectors.commands import create
23 | from forwarders.collectors.commands import delete
24 | from forwarders.collectors.commands import get
25 | from forwarders.collectors.commands import list # pylint: disable=redefined-builtin
26 | from forwarders.collectors.commands import update
27 |
28 |
29 | @click.group(name="collectors", help="Collector Management Workflows")
30 | def collectors() -> None:
31 | """Collectors group commands."""
32 | collector_dir = os.path.join(chronicle_auth.CHRONICLE_CLI_ROOT_DIR,
33 | "collectors")
34 | if not os.path.exists(collector_dir):
35 | os.mkdir(collector_dir)
36 |
37 |
38 | collectors.add_command(get.get)
39 | collectors.add_command(list.list_command)
40 | collectors.add_command(create.create)
41 | collectors.add_command(update.update)
42 | collectors.add_command(delete.delete)
43 |
--------------------------------------------------------------------------------
/forwarders/collectors/collectors_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Unit tests for collectors.py."""
16 |
17 | from click.testing import CliRunner
18 |
19 | from forwarders.collectors.collectors import collectors
20 |
21 | runner = CliRunner()
22 |
23 |
24 | def test_collectors() -> None:
25 | """Test case for collectors."""
26 | result = runner.invoke(collectors)
27 | expected_output = """Commands:
28 | create Create a collector.
29 | delete Delete a collector using forwarder and collector ID.
30 | get Get a collector using forwarder and collector ID.
31 | list List all collectors.
32 | update Update a collector using forwarder and collector ID."""
33 | assert expected_output in result.output
34 |
--------------------------------------------------------------------------------
/forwarders/collectors/commands/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 |
--------------------------------------------------------------------------------
/forwarders/collectors/commands/delete.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Delete a collector using forwarder ID and collector ID."""
16 |
17 | from typing import AnyStr
18 |
19 | import click
20 |
21 | from common import api_utility
22 | from common import chronicle_auth
23 | from common import commands_utility
24 | from common import exception_handler
25 | from common import options
26 | from common.constants import key_constants
27 | from common.constants import status
28 | from forwarders.collectors import collector_utility
29 |
30 |
31 | @click.command(
32 | name="delete", help="Delete a collector using forwarder and collector ID.")
33 | @options.url_option
34 | @options.region_option
35 | @options.verbose_option
36 | @options.credential_file_option
37 | @exception_handler.catch_exception()
38 | def delete(credential_file: AnyStr, verbose: bool, region: str,
39 | url: str) -> None:
40 | """Delete a collector using Forwarder and Collector ID with all its associated collectors.
41 |
42 | Args:
43 | credential_file (AnyStr): Path of Service Account JSON.
44 | verbose (bool): Option for printing verbose output to console.
45 | region (str): Option for selecting regions. Available options - US, EUROPE,
46 | ASIA_SOUTHEAST1.
47 | url (str): Base URL to be used for API calls.
48 |
49 | Raises:
50 | OSError: Failed to read the given file, e.g. not found, no read access
51 | (https://docs.python.org/library/exceptions.html#os-exceptions).
52 | ValueError: Invalid file contents.
53 | KeyError: Required key is not present in dictionary.
54 | TypeError: If response data is not JSON.
55 | """
56 | forwarder_id = click.prompt(
57 | "Enter Forwarder ID", default="", show_default=False)
58 | if not forwarder_id:
59 | click.echo("Forwarder ID not provided. Please enter Forwarder ID.")
60 | return
61 |
62 | collector_id = click.prompt(
63 | "Enter Collector ID", default="", show_default=False)
64 | if not collector_id:
65 | click.echo("Collector ID not provided. Please enter Collector ID.")
66 | return
67 |
68 | url = commands_utility.lower_or_none(url)
69 | client = chronicle_auth.initialize_http_session(credential_file)
70 | collector_url = f"{collector_utility.get_collector_url(region, url, forwarder_id)}/{collector_id}"
71 | method = "DELETE"
72 |
73 | collector_response = client.request(method, collector_url)
74 | status_code = collector_response.status_code
75 | collector_response = api_utility.check_content_type(collector_response.text)
76 |
77 | if status_code == status.STATUS_OK:
78 | click.echo(f"\nCollector (ID: {collector_id}) deleted successfully.")
79 | elif status_code == status.STATUS_NOT_FOUND:
80 | click.echo("Collector does not exist.")
81 | elif status_code == status.STATUS_BAD_REQUEST:
82 | click.echo("Invalid Collector ID. Please enter valid Collector ID.")
83 | else:
84 | error_message = collector_response[key_constants.KEY_ERROR][
85 | key_constants.KEY_MESSAGE]
86 | click.echo(
87 | f"\nError while fetching collector.\nResponse Code: {status_code}"
88 | f"\nError: {error_message}")
89 | return
90 |
91 | if verbose:
92 | api_utility.print_request_details(collector_url, method, None,
93 | collector_response)
94 |
--------------------------------------------------------------------------------
/forwarders/collectors/constants/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 |
--------------------------------------------------------------------------------
/forwarders/collectors/constants/schema.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Collector schema constants to be used across the project."""
16 |
--------------------------------------------------------------------------------
/forwarders/collectors/tests/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 |
--------------------------------------------------------------------------------
/forwarders/commands/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 |
--------------------------------------------------------------------------------
/forwarders/commands/create.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Create a Forwarder."""
16 |
17 | import json
18 | import os.path
19 |
20 | import click
21 |
22 | from common import api_utility
23 | from common import chronicle_auth
24 | from common import commands_utility
25 | from common import exception_handler
26 | from common import file_utility
27 | from common import options
28 | from common.constants import key_constants
29 | from common.constants import status
30 | from forwarders import forwarder_utility
31 | from forwarders import schema_utility
32 | from forwarders.collectors.commands import create as create_collector
33 | from forwarders.constants import schema
34 |
35 | CREATE_FORWARDER_BACKUP_FILE = os.path.join(
36 | chronicle_auth.CHRONICLE_CLI_ROOT_DIR, schema.KEY_FORWARDERS,
37 | "create_backup.json")
38 |
39 |
40 | @click.command(help="Create a Forwarder")
41 | @options.url_option
42 | @options.region_option
43 | @options.verbose_option
44 | @options.credential_file_option
45 | @exception_handler.catch_exception()
46 | def create(credential_file: str, verbose: bool, region: str, url: str) -> None:
47 | """Creates forwarder.
48 |
49 | Args:
50 | credential_file (AnyStr): Path of Service Account JSON.
51 | verbose (bool): Option for printing verbose output to console.
52 | region (str): Option for selecting regions. Available options - US, EUROPE,
53 | ASIA_SOUTHEAST1.
54 | url (str): Base URL to be used for API calls.
55 |
56 | Raises:
57 | OSError: Failed to read the given file, e.g. not found, no read access
58 | (https://docs.python.org/library/exceptions.html#os-exceptions).
59 | ValueError: Invalid file contents.
60 | KeyError: Required key is not present in dictionary.
61 | TypeError: If response data is not JSON.
62 | """
63 | instruction_str = (
64 | "Press Enter if you want to use the default value mentioned besides "
65 | "field description in [] brackets.")
66 | click.echo(
67 | f"{forwarder_utility.PRINT_SEPARATOR}\n{instruction_str}\n{forwarder_utility.PRINT_SEPARATOR}"
68 | )
69 |
70 | url = commands_utility.lower_or_none(url)
71 | client = chronicle_auth.initialize_http_session(credential_file)
72 | forwarder_url = forwarder_utility.get_forwarder_url(region, url)
73 | method = "POST"
74 |
75 | # Check whether backup file exists and if it exists,
76 | # read the response from the file to use it further
77 | # to show default values in prompts.
78 | backup_request_body = forwarder_utility.read_backup(
79 | CREATE_FORWARDER_BACKUP_FILE)
80 |
81 | # "backup_request_body" is received along with the response body,
82 | # for storing the existing data into the backup file.
83 | forwarder_schema = schema_utility.Schema(schema.KEY_FORWARDER_SCHEMA,
84 | backup_request_body)
85 |
86 | request_body = forwarder_schema.prepare_request_body()
87 | # Preview changes.
88 | forwarder_utility.preview_changes(request_body)
89 | selected_choice = click.confirm(
90 | "\nDo you want to create forwarder with this configuration?",
91 | default=False)
92 |
93 | if selected_choice:
94 | click.echo("\nCreating forwarder...")
95 | create_forwarder_response = client.request(method, forwarder_url,
96 | json.dumps(request_body))
97 |
98 | response = api_utility.check_content_type(create_forwarder_response.text)
99 |
100 | if create_forwarder_response.status_code != status.STATUS_OK:
101 | click.echo(
102 | "\nError occurred while creating forwarder.\nResponse Code: "
103 | f"{create_forwarder_response.status_code}.\nError: "
104 | f"{response[key_constants.KEY_ERROR][key_constants.KEY_MESSAGE]}")
105 |
106 | # "request_body" written to the backup file in case of
107 | # failure to create forwarder.
108 | forwarder_utility.write_backup(CREATE_FORWARDER_BACKUP_FILE, request_body)
109 | return
110 |
111 | forwarder_id = forwarder_utility.get_resource_id(response)
112 | click.echo(
113 | f"Forwarder created successfully with Forwarder ID: {forwarder_id}")
114 | file_utility.remove_file(CREATE_FORWARDER_BACKUP_FILE)
115 | if verbose:
116 | api_utility.print_request_details(forwarder_url, method, request_body,
117 | response)
118 |
119 | # Configure collectors for this forwarder.
120 | selected_choice = click.confirm(
121 | "\nWould you like to configure collectors for this forwarder?",
122 | default=False)
123 |
124 | while selected_choice:
125 | create_collector.create_collector(forwarder_id, url, client, verbose,
126 | region)
127 | selected_choice = click.confirm(
128 | "\nWould you like to add more collectors?", default=False)
129 |
--------------------------------------------------------------------------------
/forwarders/commands/delete.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Delete a forwarder using forwarder ID."""
16 |
17 | from typing import AnyStr
18 |
19 | import click
20 |
21 | from common import api_utility
22 | from common import chronicle_auth
23 | from common import commands_utility
24 | from common import exception_handler
25 | from common import options
26 | from common.constants import key_constants
27 | from common.constants import status
28 | from forwarders import forwarder_utility
29 |
30 |
31 | @click.command(help="Delete a forwarder using Forwarder ID")
32 | @options.url_option
33 | @options.region_option
34 | @options.verbose_option
35 | @options.credential_file_option
36 | @exception_handler.catch_exception()
37 | def delete(credential_file: AnyStr, verbose: bool, region: str,
38 | url: str) -> None:
39 | """Delete a forwarder using Forwarder ID with all its associated collectors.
40 |
41 | Args:
42 | credential_file (AnyStr): Path of Service Account JSON.
43 | verbose (bool): Option for printing verbose output to console.
44 | region (str): Option for selecting regions. Available options - US, EUROPE,
45 | ASIA_SOUTHEAST1.
46 | url (str): Base URL to be used for API calls.
47 |
48 | Raises:
49 | OSError: Failed to read the given file, e.g. not found, no read access
50 | (https://docs.python.org/library/exceptions.html#os-exceptions).
51 | ValueError: Invalid file contents.
52 | KeyError: Required key is not present in dictionary.
53 | TypeError: If response data is not JSON.
54 | """
55 |
56 | forwarder_id = click.prompt(
57 | "Enter Forwarder ID", default="", show_default=False)
58 | if not forwarder_id:
59 | click.echo("Forwarder ID not provided. Please enter Forwarder ID.")
60 | return
61 |
62 | url = commands_utility.lower_or_none(url)
63 | client = chronicle_auth.initialize_http_session(credential_file)
64 | forwarder_url = f"{forwarder_utility.get_forwarder_url(region, url)}/{forwarder_id}"
65 | method = "DELETE"
66 |
67 | click.echo("\nDeleting forwarder and all its associated collectors...")
68 | forwarder_response = client.request(method, forwarder_url)
69 | status_code = forwarder_response.status_code
70 | forwarder_response = api_utility.check_content_type(forwarder_response.text)
71 |
72 | if status_code == status.STATUS_OK:
73 | click.echo(
74 | f"\nForwarder (ID: {forwarder_id}) deleted successfully with all its associated collectors."
75 | )
76 | elif status_code == status.STATUS_NOT_FOUND:
77 | click.echo("Forwarder does not exist.")
78 | elif status_code == status.STATUS_BAD_REQUEST:
79 | click.echo("Invalid Forwarder ID. Please enter valid Forwarder ID.")
80 | else:
81 | if status_code != status.STATUS_OK:
82 | error_message = forwarder_response[key_constants.KEY_ERROR][
83 | key_constants.KEY_MESSAGE]
84 | click.echo(
85 | f"\nError while fetching forwarder.\nResponse Code: {status_code}"
86 | f"\nError: {error_message}")
87 | return
88 |
89 | if verbose:
90 | api_utility.print_request_details(forwarder_url, method, None,
91 | forwarder_response)
92 |
--------------------------------------------------------------------------------
/forwarders/commands/generate_files.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Generate Forwarders Files."""
16 |
17 | import os
18 | from typing import AnyStr
19 |
20 | import click
21 |
22 | from common import api_utility
23 | from common import chronicle_auth
24 | from common import commands_utility
25 | from common import exception_handler
26 | from common import file_utility
27 | from common import options
28 | from common.constants import key_constants
29 | from common.constants import status
30 | from forwarders import forwarder_utility
31 | from forwarders.constants import schema
32 |
33 | CONF_FILE_EXTENSION = "conf"
34 |
35 |
36 | @click.command(
37 | name="generate_files",
38 | help="Generate forwarder configuration files using Forwarder ID")
39 | @options.url_option
40 | @options.region_option
41 | @options.verbose_option
42 | @options.credential_file_option
43 | @click.option(
44 | "-f",
45 | "--file-path",
46 | default="",
47 | help="Download generated forwarder files to the specified path.")
48 | @exception_handler.catch_exception()
49 | def generate_files(credential_file: AnyStr, verbose: bool, region: str,
50 | url: str, file_path: AnyStr) -> None:
51 | """Generate forwarder files using Forwarder ID.
52 |
53 | Args:
54 | credential_file (AnyStr): Path of Service Account JSON.
55 | verbose (bool): Option for printing listverbose output to console.
56 | region (str): Option for selecting regions. Available options - US, EUROPE,
57 | ASIA_SOUTHEAST1.
58 | url (str): Base URL to be used for API calls.
59 | file_path (AnyStr): Path where the generated forwarder files would be
60 | stored.
61 | """
62 | forwarder_id = click.prompt(
63 | "Enter Forwarder ID", default="", show_default=False)
64 | if not forwarder_id:
65 | click.echo("Forwarder ID not provided. Please enter Forwarder ID.")
66 | return
67 |
68 | url = commands_utility.lower_or_none(url)
69 | client = chronicle_auth.initialize_http_session(credential_file)
70 | forwarder_url = f"{forwarder_utility.get_forwarder_url(region, url)}/{forwarder_id}:generateForwarderFiles"
71 | method = "GET"
72 |
73 | generate_forwarders_response = client.request(method, forwarder_url)
74 | forwarder_response = api_utility.check_content_type(
75 | generate_forwarders_response.text)
76 | status_code = generate_forwarders_response.status_code
77 |
78 | download_path = os.path.abspath(file_path) if file_path else os.path.abspath(
79 | forwarder_id)
80 |
81 | click.echo("Generating forwarder files ...")
82 |
83 | if status_code == status.STATUS_OK:
84 | file_utility.export_txt(f"{download_path}_forwarder.{CONF_FILE_EXTENSION}",
85 | forwarder_response[schema.KEY_CONFIG])
86 | file_utility.export_txt(
87 | f"{download_path}_forwarder_auth.{CONF_FILE_EXTENSION}",
88 | forwarder_response[schema.KEY_AUTH])
89 | click.echo(
90 | f"Forwarder files generated successfully.\nConfiguration file: {download_path}_forwarder.{CONF_FILE_EXTENSION}\nAuth file: {download_path}_forwarder_auth.{CONF_FILE_EXTENSION}"
91 | )
92 | else:
93 | error_message = forwarder_response[key_constants.KEY_ERROR][
94 | key_constants.KEY_MESSAGE]
95 | click.echo(
96 | f"\nError while generating forwarder files.\nResponse Code: {status_code}"
97 | f"\nError: {error_message}")
98 |
99 | if verbose:
100 | api_utility.print_request_details(forwarder_url, method, None,
101 | forwarder_response)
102 |
--------------------------------------------------------------------------------
/forwarders/constants/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 |
--------------------------------------------------------------------------------
/forwarders/forwarder_templates.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Templates for printing output to console."""
16 |
17 | import string
18 |
19 | header_template = string.Template("""
20 | ========================================
21 | ${display_name}
22 | ========================================""")
23 |
24 | preview_template = string.Template("""\
25 | Preview changes:
26 |
27 | - Press Up/b or Down/z keys to paginate.
28 | - To switch case-sensitivity, press '-i' and press enter. By default, search
29 | is case-sensitive.
30 | - To search for specific field, press '/' key, enter text and press enter.
31 | - Press 'q' to quit and confirm preview changes.
32 | - Press `h` for all the available options to navigate the list.
33 | =============================================================================
34 | """)
35 |
36 | preview_template_win = string.Template("""\
37 | Preview changes:
38 |
39 | - Press ENTER key (scrolls one line at a time) or SPACEBAR key (display next screen).
40 | - Press 'q' to quit and confirm preview changes.
41 | =============================================================================
42 | """)
43 |
44 | retry_template = string.Template("""
45 | Looks like there was a failed create/update attempt for ${display_name}.
46 | Would you like to retry?
47 | """)
48 |
--------------------------------------------------------------------------------
/forwarders/forwarders.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Grouping Forwarders CLI commands."""
16 |
17 | import os
18 |
19 | import click
20 |
21 | from common import chronicle_auth
22 | from forwarders.collectors.collectors import collectors
23 | from forwarders.commands import create
24 | from forwarders.commands import delete
25 | from forwarders.commands import generate_files
26 | from forwarders.commands import get
27 | from forwarders.commands import list # pylint: disable=redefined-builtin
28 | from forwarders.commands import update
29 |
30 |
31 | @click.group(name="forwarders", help="Forwarder Management Workflows")
32 | def forwarders() -> None:
33 | """Forwarders group commands."""
34 | forwarder_dir = os.path.join(chronicle_auth.CHRONICLE_CLI_ROOT_DIR,
35 | "forwarders")
36 | if not os.path.exists(forwarder_dir):
37 | os.mkdir(forwarder_dir)
38 |
39 |
40 | forwarders.add_command(collectors)
41 | forwarders.add_command(get.get)
42 | forwarders.add_command(list.list_command)
43 | forwarders.add_command(create.create)
44 | forwarders.add_command(update.update)
45 | forwarders.add_command(delete.delete)
46 | forwarders.add_command(generate_files.generate_files)
47 |
--------------------------------------------------------------------------------
/forwarders/forwarders_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Unit tests for forwarders.py."""
16 |
17 | from click.testing import CliRunner
18 |
19 | from forwarders.forwarders import forwarders
20 |
21 | runner = CliRunner()
22 |
23 |
24 | def test_forwarders() -> None:
25 | """Test case for forwarders."""
26 | result = runner.invoke(forwarders)
27 | expected_output = """Commands:
28 | collectors Collector Management Workflows
29 | create Create a Forwarder
30 | delete Delete a forwarder using Forwarder ID
31 | generate_files Generate forwarder configuration files using Forwarder ID
32 | get Get forwarder details using Forwarder ID
33 | list List all forwarders
34 | update Update a forwarder using Forwarder ID"""
35 | assert expected_output in result.output
36 |
--------------------------------------------------------------------------------
/forwarders/schemas/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 |
--------------------------------------------------------------------------------
/forwarders/tests/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 |
--------------------------------------------------------------------------------
/main.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Starting point for the Chronicle CLI."""
16 |
17 | import os
18 | import subprocess
19 |
20 | import click
21 | from click._compat import WIN
22 |
23 | from common import chronicle_auth
24 | from feeds.feeds import feeds
25 | from forwarders.forwarders import forwarders
26 | from parsers.parsers import parsers
27 | from tools.bigquery import bigquery
28 |
29 |
30 | @click.group(
31 | name="cli",
32 | context_settings=dict(help_option_names=["-h", "--help"]),
33 | help="Chronicle CLI is a CLI tool for managing Chronicle user workflows for e.g. Feed Management workflows."
34 | )
35 | def cli() -> None:
36 | """Chronicle CLI commands."""
37 | if not os.path.exists(chronicle_auth.CHRONICLE_CLI_ROOT_DIR):
38 | click.echo(
39 | "'~/.chronicle_cli' directory is not present.\nCreating directory...")
40 | os.mkdir(chronicle_auth.CHRONICLE_CLI_ROOT_DIR)
41 | if WIN:
42 | subprocess.call(["attrib", "+H", chronicle_auth.CHRONICLE_CLI_ROOT_DIR])
43 | click.echo(
44 | f"Directory '{chronicle_auth.CHRONICLE_CLI_ROOT_DIR}' created successfully."
45 | )
46 |
47 |
48 | cli.add_command(feeds)
49 | cli.add_command(forwarders)
50 | cli.add_command(parsers)
51 | cli.add_command(bigquery)
52 |
53 | if __name__ == "__main__":
54 | cli()
55 |
--------------------------------------------------------------------------------
/main_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Unit tests for main.py."""
16 | from click.testing import CliRunner
17 | from main import cli
18 |
19 | runner = CliRunner()
20 |
21 |
22 | def test_main() -> None:
23 | """Test case for main."""
24 | result = runner.invoke(cli)
25 | assert """Usage: cli [OPTIONS] COMMAND [ARGS]...
26 |
27 | Chronicle CLI is a CLI tool for managing Chronicle user workflows for e.g.
28 | Feed Management workflows.
29 |
30 | Options:
31 | -h, --help Show this message and exit.
32 |
33 | Commands:
34 | bigquery Manage Big Query export
35 | feeds Feed Management Workflows
36 | forwarders Forwarder Management Workflows
37 | parsers Manage config based parsers
38 | """ == result.output
39 |
--------------------------------------------------------------------------------
/mock_test_utility.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Utility classes or functions for tests."""
16 |
17 |
18 | class MockResponse:
19 | """Mock response.
20 |
21 | Attributes:
22 | status_code: Response status code
23 | text: Response content
24 | """
25 |
26 | def __init__(self, status_code: int, text: str):
27 | self.status_code = status_code
28 | self.text = text
29 |
--------------------------------------------------------------------------------
/parsers/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 |
--------------------------------------------------------------------------------
/parsers/commands/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 |
--------------------------------------------------------------------------------
/parsers/commands/activate_parser.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Activate a parser."""
16 |
17 | import click
18 |
19 | from common import api_utility
20 | from common import chronicle_auth
21 | from common import exception_handler
22 | from common import options
23 | from common.constants import key_constants as common_constants
24 | from common.constants import status
25 | from parsers import url
26 |
27 |
28 | @click.command(name="activate_parser", help="[New]Activate a parser")
29 | @click.argument("project_id", required=True, default="")
30 | @click.argument("customer_id", required=True, default="")
31 | @click.argument("log_type", required=True, default="")
32 | @click.argument("parser_id", required=True, default="")
33 | @options.env_option
34 | @options.region_option
35 | @options.verbose_option
36 | @options.credential_file_option
37 | @options.v2_option
38 | @exception_handler.catch_exception()
39 | def activate_parser(
40 | v2: bool,
41 | credential_file: str,
42 | verbose: bool,
43 | region: str,
44 | env: str,
45 | project_id: str,
46 | customer_id: str,
47 | log_type: str,
48 | parser_id: int) -> None:
49 | """Activate a parser given the Parser ID.
50 |
51 | Args:
52 | v2 (bool): Option for enabling v2 commands.
53 | credential_file (AnyStr): Path of Service Account JSON.
54 | verbose (bool): Option for printing verbose output to console.
55 | region (str): Option for selecting regions. Available options - US, EUROPE,
56 | ASIA_SOUTHEAST1.
57 | env (str): Option for selection environment. Available options - prod, test.
58 | project_id (str): The GCP Project ID.
59 | customer_id (str): The Customer ID.
60 | log_type (str): The Log Type.
61 | parser_id (int): The Parser ID.
62 |
63 | Raises:
64 | OSError: Failed to read the given file, e.g. not found, no read access
65 | (https://docs.python.org/library/exceptions.html#os-exceptions).
66 | ValueError: Invalid file contents.
67 | KeyError: Required key is not present in dictionary.
68 | TypeError: If response data is not JSON.
69 | """
70 | if not v2:
71 | click.echo("--v2 flag not provided. "
72 | "Please provide the flag to run the new commands")
73 | return
74 |
75 | if not project_id:
76 | click.echo("Project ID not provided. Please enter Project ID")
77 | return
78 |
79 | if not customer_id:
80 | click.echo("Customer ID not provided. Please enter Customer ID")
81 | return
82 |
83 | if not log_type:
84 | click.echo("Log Type not provided. Please enter Log Type")
85 | return
86 |
87 | if not parser_id:
88 | click.echo("Parser ID not provided. Please enter Parser ID")
89 | return
90 |
91 | click.echo("Activating Parser...")
92 |
93 | resources = {
94 | "project": project_id,
95 | "location": region.lower(),
96 | "instance": customer_id,
97 | "log_type": log_type,
98 | "parser": parser_id
99 | }
100 |
101 | activate_parser_url = url.get_dataplane_url(
102 | region,
103 | "activate_parser",
104 | env,
105 | resources)
106 | client = chronicle_auth.initialize_dataplane_http_session(credential_file)
107 | method = "POST"
108 | response = client.request(
109 | method, activate_parser_url, timeout=url.HTTP_REQUEST_TIMEOUT_IN_SECS)
110 | parsed_response = api_utility.check_content_type(response.text)
111 |
112 | if response.status_code != status.STATUS_OK:
113 | click.echo(
114 | f"Error while activating parser.\n"
115 | f"Response Code: {response.status_code}\n"
116 | f"Error: "
117 | f"{parsed_response[common_constants.KEY_ERROR][common_constants.KEY_MESSAGE]}"
118 | )
119 | return
120 |
121 | click.echo("Parser activated successfully.")
122 |
123 | if verbose:
124 | api_utility.print_request_details(
125 | activate_parser_url, method, None, parsed_response
126 | )
127 |
--------------------------------------------------------------------------------
/parsers/commands/archive.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Archives a parser given the config ID."""
16 |
17 | from typing import AnyStr
18 |
19 | import click
20 |
21 | from common import api_utility
22 | from common import chronicle_auth
23 | from common import exception_handler
24 | from common import options
25 | from common.constants import key_constants as common_constants
26 | from common.constants import status
27 | from parsers import parser_templates
28 | from parsers import url
29 | from parsers.constants import key_constants as parser_constants
30 |
31 |
32 | @click.command(name="archive", help="Archives a parser given the config ID")
33 | @options.env_option
34 | @options.region_option
35 | @options.verbose_option
36 | @options.credential_file_option
37 | @exception_handler.catch_exception()
38 | def archive(credential_file: AnyStr, verbose: bool, region: str,
39 | env: str) -> None:
40 | """Archives a parser given the config ID.
41 |
42 | Args:
43 | credential_file (AnyStr): Path of Service Account JSON.
44 | verbose (bool): Option for printing verbose output to console.
45 | region (str): Option for selecting regions. Available options - US, EUROPE,
46 | ASIA_SOUTHEAST1.
47 | env (str): Option for selection environment. Available options - prod, test.
48 |
49 | Raises:
50 | OSError: Failed to read the given file, e.g. not found, no read access
51 | (https://docs.python.org/library/exceptions.html#os-exceptions).
52 | ValueError: Invalid file contents.
53 | KeyError: Required key is not present in dictionary.
54 | TypeError: If response data is not JSON.
55 | """
56 | config_id = click.prompt("Enter Config ID", show_default=False, default="")
57 |
58 | if not config_id:
59 | click.echo("Config ID not provided. Please enter Config ID.")
60 | return
61 |
62 | click.echo("Archiving parser...")
63 |
64 | archive_parser_url = f"{url.get_url(region, 'list', env)}/{config_id}:archive"
65 | client = chronicle_auth.initialize_http_session(credential_file)
66 | method = "POST"
67 | response = client.request(
68 | method, archive_parser_url, timeout=url.HTTP_REQUEST_TIMEOUT_IN_SECS)
69 |
70 | parsed_response = api_utility.check_content_type(response.text)
71 |
72 | if response.status_code != status.STATUS_OK:
73 | click.echo(
74 | f"Error while archiving parser.\nResponse Code: {response.status_code}"
75 | f"\nError: {parsed_response[common_constants.KEY_ERROR][common_constants.KEY_MESSAGE]}"
76 | )
77 | return
78 |
79 | click.echo("\nParser archived successfully.")
80 | parser_details = parser_templates.parser_details_template.substitute(
81 | config_id=f"{parsed_response[parser_constants.KEY_CONFIG_ID]}",
82 | log_type=f"{parsed_response[common_constants.KEY_LOG_TYPE]}",
83 | state=f"{parsed_response[parser_constants.KEY_STATE]}",
84 | sha256=f"{parsed_response[parser_constants.KEY_SHA256]}",
85 | author=f"{parsed_response[parser_constants.KEY_AUTHOR]}",
86 | submit_time=f"{parsed_response[parser_constants.KEY_SUBMIT_TIME]}",
87 | last_live_time=f'{parsed_response.get(parser_constants.KEY_LAST_LIVE_TIME, "-")}',
88 | state_last_changed_time=f"{parsed_response[parser_constants.KEY_STATE_LAST_CHANGED_TIME]}"
89 | )
90 | click.echo(parser_details)
91 |
92 | if verbose:
93 | api_utility.print_request_details(archive_parser_url, method, None,
94 | parsed_response)
95 |
--------------------------------------------------------------------------------
/parsers/commands/archive_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Tests for archive.py."""
16 |
17 | from unittest import mock
18 |
19 | from click.testing import CliRunner
20 |
21 | from common import uri
22 | from mock_test_utility import MockResponse
23 | from parsers import url
24 | from parsers.commands.archive import archive
25 | from parsers.tests.fixtures import * # pylint: disable=wildcard-import
26 |
27 |
28 | runner = CliRunner()
29 | TEST_ARCHIVE_URL = f"{uri.BASE_URL}/tools/cbnParsers/test_config_id:archive"
30 |
31 |
32 | @mock.patch(
33 | "common.chronicle_auth.initialize_http_session"
34 | )
35 | @mock.patch("parsers.url.get_url")
36 | @mock.patch(
37 | "parsers.commands.archive.click.prompt")
38 | def test_archive_parser(mock_input: mock.MagicMock, mock_url: mock.MagicMock,
39 | mock_client: mock.MagicMock,
40 | test_archive_data: MockResponse) -> None:
41 | """Test case to check response for status of parser.
42 |
43 | Args:
44 | mock_input (mock.MagicMock): Mock prompt object
45 | mock_url (mock.MagicMock): Mock object
46 | mock_client (mock.MagicMock): Mock object
47 | test_archive_data (Tuple): Test input data
48 | """
49 | mock_url.return_value = f"{uri.BASE_URL}/tools/cbnParsers"
50 | mock_input.side_effect = ["test_config_id"]
51 | mock_client.return_value = mock.Mock()
52 | mock_client.return_value.request.side_effect = [test_archive_data]
53 | result = runner.invoke(archive)
54 | assert """Archiving parser...
55 |
56 | Parser archived successfully.
57 |
58 | Parser Details:
59 | Config ID: test_config_id
60 | Log type: TEST_LOG_TYPE
61 | State: ARCHIVED
62 | SHA256: test_sha256
63 | Author: test_user
64 | Submit Time: 2022-04-01T08:08:44.217797Z
65 | State Last Changed Time: 2022-04-01T08:08:44.217797Z
66 | Last Live Time: 2022-04-01T08:08:44.217797Z
67 | """ in result.output
68 | mock_url.assert_called_once_with("US", "list", "prod")
69 | mock_client.return_value.request.assert_called_once_with(
70 | "POST", TEST_ARCHIVE_URL, timeout=url.HTTP_REQUEST_TIMEOUT_IN_SECS)
71 |
72 |
73 | @mock.patch(
74 | "common.chronicle_auth.initialize_http_session"
75 | )
76 | @mock.patch("parsers.url.get_url")
77 | @mock.patch(
78 | "parsers.commands.archive.click.prompt")
79 | def test_archive_parser_500(mock_input: mock.MagicMock,
80 | mock_url: mock.MagicMock,
81 | mock_client: mock.MagicMock,
82 | test_500_resp: MockResponse) -> None:
83 | """Test case to check response for archive parser 500 response code.
84 |
85 | Args:
86 | mock_input (mock.MagicMock): Mock prompt object
87 | mock_url (mock.MagicMock): Mock object
88 | mock_client (mock.MagicMock): Mock object
89 | test_500_resp (Tuple): Test response data
90 | """
91 | mock_input.side_effect = ["test_config_id"]
92 | mock_url.return_value = TEST_ARCHIVE_URL
93 | mock_client.return_value = mock.Mock()
94 | mock_client.return_value.request.side_effect = [test_500_resp]
95 | result = runner.invoke(archive)
96 | assert """Archiving parser...
97 | Error while archiving parser.
98 | Response Code: 500
99 | Error: test error
100 | """ in result.output
101 |
102 |
103 | @mock.patch(
104 | "parsers.commands.history.click.prompt")
105 | def test_archive_empty_log_type(mock_input: mock.MagicMock) -> None:
106 | """Test case to check the console output if no input provided for log type.
107 |
108 | Args:
109 | mock_input (mock.MagicMock): Mock object
110 | """
111 | mock_input.return_value = ""
112 | result = runner.invoke(archive)
113 | assert """Config ID not provided. Please enter Config ID.""" in result.output
114 |
115 |
116 | def test_prompt_text() -> None:
117 | """Test case to check prompt text."""
118 | result = runner.invoke(archive)
119 | assert "Enter Config ID:" in result.output
120 |
--------------------------------------------------------------------------------
/parsers/commands/classify_log_type.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Classify the provided logs to the corresponding log types."""
16 |
17 | import base64
18 | import os
19 |
20 | import click
21 |
22 | from common import api_utility
23 | from common import chronicle_auth
24 | from common import exception_handler
25 | from common import options
26 | from common.constants import key_constants as common_constants
27 | from common.constants import status
28 | from parsers import url
29 | from parsers.constants import key_constants as parser_constants
30 |
31 |
32 | @click.command(
33 | name="classify_log_type",
34 | help="[New]Classify the provided logs to the log types.")
35 | @click.argument("project_id", required=True, default="")
36 | @click.argument("customer_id", required=True, default="")
37 | @click.argument("log_file", required=True, default="")
38 | @options.env_option
39 | @options.region_option
40 | @options.verbose_option
41 | @options.credential_file_option
42 | @options.v2_option
43 | @exception_handler.catch_exception()
44 | def classify_log_type(
45 | v2: bool,
46 | credential_file: str,
47 | verbose: bool,
48 | region: str,
49 | env: str,
50 | project_id: str,
51 | customer_id: str,
52 | log_file: str) -> None:
53 | """Classify the provided logs to the corresponding log types.
54 |
55 | Args:
56 | v2 (bool): Option for enabling v2 commands.
57 | credential_file (AnyStr): Path of Service Account JSON.
58 | verbose (bool): Option for printing verbose output to console.
59 | region (str): Option for selecting regions. Available options - US, EUROPE,
60 | ASIA_SOUTHEAST1.
61 | env (str): Option for selection environment. Available options - prod, test.
62 | project_id (str): The GCP Project ID.
63 | customer_id (str): The Customer ID.
64 | log_file (str): Path of log file containing a single log line.
65 |
66 | Raises:
67 | OSError: Failed to read the given file, e.g. not found, no read access
68 | (https://docs.python.org/library/exceptions.html#os-exceptions).
69 | ValueError: Invalid file contents.
70 | KeyError: Required key is not present in dictionary.
71 | TypeError: If response data is not JSON.
72 | """
73 | if not v2:
74 | click.echo("--v2 flag not provided. "
75 | "Please provide the flag to run the new commands")
76 | return
77 |
78 | if not project_id:
79 | click.echo("Project ID not provided. Please enter Project ID")
80 | return
81 |
82 | if not customer_id:
83 | click.echo("Customer ID not provided. Please enter Customer ID")
84 | return
85 |
86 | if not os.path.exists(log_file):
87 | click.echo(f"{log_file} does not exist. "
88 | "Please enter valid log file path")
89 | return
90 |
91 | click.echo("Classifying the provided log to the corresponding log types...\n")
92 |
93 | resources = {
94 | "project": project_id,
95 | "location": region.lower(),
96 | "instance": customer_id
97 | }
98 |
99 | with open(log_file, "r") as f:
100 | log_lines = f.readlines()
101 |
102 | log_data = []
103 | for log_line in log_lines:
104 | log_line = log_line.strip(" \n")
105 | log_data.append(base64.b64encode(log_line.encode()).decode())
106 |
107 | data = {
108 | parser_constants.KEY_LOG_DATA: log_data,
109 | }
110 |
111 | classify_log_type_url = url.get_dataplane_url(
112 | region,
113 | "classify_log_type",
114 | env,
115 | resources)
116 | client = chronicle_auth.initialize_dataplane_http_session(credential_file)
117 | method = "POST"
118 | response = client.request(
119 | method, classify_log_type_url,
120 | json=data, timeout=url.HTTP_REQUEST_TIMEOUT_IN_SECS)
121 | parsed_response = api_utility.check_content_type(response.text)
122 |
123 | if response.status_code != status.STATUS_OK:
124 | click.echo(
125 | f"Error while classifying the logs.\n"
126 | f"Response Code: {response.status_code}\n"
127 | f"Error: "
128 | f"{parsed_response[common_constants.KEY_ERROR][common_constants.KEY_MESSAGE]}"
129 | )
130 | return
131 |
132 | if parser_constants.KEY_PREDICTIONS not in parsed_response:
133 | click.echo("No predictions found in the response.")
134 | return
135 |
136 | results = parsed_response.get(parser_constants.KEY_PREDICTIONS, [])
137 | for result in results:
138 | # Handle log type and score
139 | log_type = result[parser_constants.KEY_LOGTYPE]
140 | score = result[parser_constants.KEY_SCORE]
141 | click.echo(f"Log Type: {log_type} , Score: {score}")
142 |
143 | if verbose:
144 | api_utility.print_request_details(
145 | classify_log_type_url, method, None, parsed_response)
146 |
--------------------------------------------------------------------------------
/parsers/commands/deactivate_parser.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Deactivate a parser."""
16 |
17 | import click
18 |
19 | from common import api_utility
20 | from common import chronicle_auth
21 | from common import exception_handler
22 | from common import options
23 | from common.constants import key_constants as common_constants
24 | from common.constants import status
25 | from parsers import url
26 |
27 |
28 | @click.command(name="deactivate_parser", help="[New]Deactivate a parser")
29 | @click.argument("project_id", required=True, default="")
30 | @click.argument("customer_id", required=True, default="")
31 | @click.argument("log_type", required=True, default="")
32 | @click.argument("parser_id", required=True, default="")
33 | @options.env_option
34 | @options.region_option
35 | @options.verbose_option
36 | @options.credential_file_option
37 | @options.v2_option
38 | @exception_handler.catch_exception()
39 | def deactivate_parser(
40 | v2: bool,
41 | credential_file: str,
42 | verbose: bool,
43 | region: str,
44 | env: str,
45 | project_id: str,
46 | customer_id: str,
47 | log_type: str,
48 | parser_id: int) -> None:
49 | """Deactivate a parser given the Parser ID.
50 |
51 | Args:
52 | v2 (bool): Option for enabling v2 commands.
53 | credential_file (AnyStr): Path of Service Account JSON.
54 | verbose (bool): Option for printing verbose output to console.
55 | region (str): Option for selecting regions. Available options - US, EUROPE,
56 | ASIA_SOUTHEAST1.
57 | env (str): Option for selection environment. Available options - prod, test.
58 | project_id (str): The GCP Project ID.
59 | customer_id (str): The Customer ID.
60 | log_type (str): The Log Type.
61 | parser_id (int): The Parser ID.
62 |
63 | Raises:
64 | OSError: Failed to read the given file, e.g. not found, no read access
65 | (https://docs.python.org/library/exceptions.html#os-exceptions).
66 | ValueError: Invalid file contents.
67 | KeyError: Required key is not present in dictionary.
68 | TypeError: If response data is not JSON.
69 | """
70 | if not v2:
71 | click.echo("--v2 flag not provided. "
72 | "Please provide the flag to run the new commands")
73 | return
74 |
75 | if not project_id:
76 | click.echo("Project ID not provided. Please enter Project ID")
77 | return
78 |
79 | if not customer_id:
80 | click.echo("Customer ID not provided. Please enter Customer ID")
81 | return
82 |
83 | if not log_type:
84 | click.echo("Log Type not provided. Please enter Log Type")
85 | return
86 |
87 | if not parser_id:
88 | click.echo("Parser ID not provided. Please enter Parser ID")
89 | return
90 |
91 | click.echo("Deactivating Parser...")
92 |
93 | resources = {
94 | "project": project_id,
95 | "location": region.lower(),
96 | "instance": customer_id,
97 | "log_type": log_type,
98 | "parser": parser_id
99 | }
100 |
101 | deactivate_parser_url = url.get_dataplane_url(
102 | region,
103 | "deactivate_parser",
104 | env,
105 | resources)
106 | client = chronicle_auth.initialize_dataplane_http_session(credential_file)
107 | method = "POST"
108 | response = client.request(
109 | method, deactivate_parser_url, timeout=url.HTTP_REQUEST_TIMEOUT_IN_SECS)
110 | parsed_response = api_utility.check_content_type(response.text)
111 |
112 | if response.status_code != status.STATUS_OK:
113 | click.echo(
114 | f"Error while deactivating parser.\n"
115 | f"Response Code: {response.status_code}\n"
116 | f"Error: "
117 | f"{parsed_response[common_constants.KEY_ERROR][common_constants.KEY_MESSAGE]}"
118 | )
119 | return
120 |
121 | click.echo("Parser deactivated successfully.")
122 |
123 | if verbose:
124 | api_utility.print_request_details(
125 | deactivate_parser_url, method, None, parsed_response
126 | )
127 |
--------------------------------------------------------------------------------
/parsers/commands/delete_extension.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Delete an extension."""
16 |
17 | import click
18 |
19 | from common import api_utility
20 | from common import chronicle_auth
21 | from common import exception_handler
22 | from common import options
23 | from common.constants import key_constants as common_constants
24 | from common.constants import status
25 | from parsers import url
26 |
27 |
28 | @click.command(name="delete_extension", help="[New]Delete an extension")
29 | @click.argument("project_id", required=True, default="")
30 | @click.argument("customer_id", required=True, default="")
31 | @click.argument("log_type", required=True, default="")
32 | @click.argument("parserextension_id", required=True, default="")
33 | @options.env_option
34 | @options.region_option
35 | @options.verbose_option
36 | @options.credential_file_option
37 | @options.v2_option
38 | @exception_handler.catch_exception()
39 | def delete_extension(
40 | v2: bool,
41 | credential_file: str,
42 | verbose: bool,
43 | region: str,
44 | env: str,
45 | project_id: str,
46 | customer_id: str,
47 | log_type: str,
48 | parserextension_id: str) -> None:
49 | """Delete a parser extension given the ParserExtension ID.
50 |
51 | Args:
52 | v2 (bool): Option for enabling v2 commands.
53 | credential_file (AnyStr): Path of Service Account JSON.
54 | verbose (bool): Option for printing verbose output to console.
55 | region (str): Option for selecting regions. Available options - US, EUROPE,
56 | ASIA_SOUTHEAST1.
57 | env (str): Option for selection environment. Available options - prod, test.
58 | project_id (str): The GCP Project ID.
59 | customer_id (str): The Customer ID.
60 | log_type (str): The Log Type.
61 | parserextension_id (str): The ParserExtension ID.
62 |
63 | Raises:
64 | OSError: Failed to read the given file, e.g. not found, no read access
65 | (https://docs.python.org/library/exceptions.html#os-exceptions).
66 | ValueError: Invalid file contents.
67 | KeyError: Required key is not present in dictionary.
68 | TypeError: If response data is not JSON.
69 | """
70 | if not v2:
71 | click.echo("--v2 flag not provided. "
72 | "Please provide the flag to run the new commands")
73 | return
74 |
75 | if not project_id:
76 | click.echo("Project ID not provided. Please enter Project ID")
77 | return
78 |
79 | if not customer_id:
80 | click.echo("Customer ID not provided. Please enter Customer ID")
81 | return
82 |
83 | if not log_type:
84 | click.echo("Log Type not provided. Please enter Log Type")
85 | return
86 |
87 | if not parserextension_id:
88 | click.echo("ParserExtension ID not provided. "
89 | "Please enter ParserExtension ID")
90 | return
91 |
92 | click.echo("Deleting Parser Extension...")
93 |
94 | resources = {
95 | "project": project_id,
96 | "location": region.lower(),
97 | "instance": customer_id,
98 | "log_type": log_type,
99 | "parser_extension": parserextension_id,
100 | }
101 |
102 | delete_extension_url = url.get_dataplane_url(
103 | region,
104 | "delete_extension",
105 | env,
106 | resources)
107 | client = chronicle_auth.initialize_dataplane_http_session(credential_file)
108 | method = "DELETE"
109 | response = client.request(
110 | method, delete_extension_url, timeout=url.HTTP_REQUEST_TIMEOUT_IN_SECS)
111 | parsed_response = api_utility.check_content_type(response.text)
112 |
113 | if response.status_code != status.STATUS_OK:
114 | click.echo(
115 | f"Error while deleting parser extension.\n"
116 | f"Response Code: {response.status_code}\n"
117 | f"Error: "
118 | f"{parsed_response[common_constants.KEY_ERROR][common_constants.KEY_MESSAGE]}"
119 | )
120 | return
121 |
122 | click.echo("Parser Extension deleted successfully.")
123 |
124 | if verbose:
125 | api_utility.print_request_details(
126 | delete_extension_url, method, None, parsed_response
127 | )
128 |
--------------------------------------------------------------------------------
/parsers/commands/delete_parser.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Delete a parser."""
16 |
17 | import click
18 |
19 | from common import api_utility
20 | from common import chronicle_auth
21 | from common import exception_handler
22 | from common import options
23 | from common.constants import key_constants as common_constants
24 | from common.constants import status
25 | from parsers import url
26 |
27 |
28 | @click.command(name="delete_parser", help="[New]Delete a parser")
29 | @click.argument("project_id", required=True, default="")
30 | @click.argument("customer_id", required=True, default="")
31 | @click.argument("log_type", required=True, default="")
32 | @click.argument("parser_id", required=True, default="")
33 | @options.env_option
34 | @options.region_option
35 | @options.verbose_option
36 | @options.credential_file_option
37 | @options.v2_option
38 | @exception_handler.catch_exception()
39 | def delete_parser(
40 | v2: bool,
41 | credential_file: str,
42 | verbose: bool,
43 | region: str,
44 | env: str,
45 | project_id: str,
46 | customer_id: str,
47 | log_type: str,
48 | parser_id: int) -> None:
49 | """Delete a parser given the Parser ID.
50 |
51 | Args:
52 | v2 (bool): Option for enabling v2 commands.
53 | credential_file (AnyStr): Path of Service Account JSON.
54 | verbose (bool): Option for printing verbose output to console.
55 | region (str): Option for selecting regions. Available options - US, EUROPE,
56 | ASIA_SOUTHEAST1.
57 | env (str): Option for selection environment. Available options - prod, test.
58 | project_id (str): The GCP Project ID.
59 | customer_id (str): The Customer ID.
60 | log_type (str): The Log Type.
61 | parser_id (int): The Parser ID.
62 |
63 | Raises:
64 | OSError: Failed to read the given file, e.g. not found, no read access
65 | (https://docs.python.org/library/exceptions.html#os-exceptions).
66 | ValueError: Invalid file contents.
67 | KeyError: Required key is not present in dictionary.
68 | TypeError: If response data is not JSON.
69 | """
70 | if not v2:
71 | click.echo("--v2 flag not provided. "
72 | "Please provide the flag to run the new commands")
73 | return
74 |
75 | if not project_id:
76 | click.echo("Project ID not provided. Please enter Project ID")
77 | return
78 |
79 | if not customer_id:
80 | click.echo("Customer ID not provided. Please enter Customer ID")
81 | return
82 |
83 | if not log_type:
84 | click.echo("Log Type not provided. Please enter Log Type")
85 | return
86 |
87 | if not parser_id:
88 | click.echo("Parser ID not provided. Please enter Parser ID")
89 | return
90 |
91 | click.echo("Deleting Parser...")
92 |
93 | resources = {
94 | "project": project_id,
95 | "location": region.lower(),
96 | "instance": customer_id,
97 | "log_type": log_type,
98 | "parser": parser_id
99 | }
100 |
101 | delete_parser_url = url.get_dataplane_url(
102 | region,
103 | "delete_parser",
104 | env,
105 | resources)
106 | client = chronicle_auth.initialize_dataplane_http_session(credential_file)
107 | method = "DELETE"
108 | response = client.request(
109 | method, delete_parser_url, timeout=url.HTTP_REQUEST_TIMEOUT_IN_SECS)
110 | parsed_response = api_utility.check_content_type(response.text)
111 |
112 | if response.status_code != status.STATUS_OK:
113 | click.echo(
114 | f"Error while deleting parser.\n"
115 | f"Response Code: {response.status_code}\n"
116 | f"Error: "
117 | f"{parsed_response[common_constants.KEY_ERROR][common_constants.KEY_MESSAGE]}"
118 | )
119 | return
120 |
121 | click.echo("Parser deleted successfully.")
122 |
123 | if verbose:
124 | api_utility.print_request_details(
125 | delete_parser_url, method, None, parsed_response
126 | )
127 |
--------------------------------------------------------------------------------
/parsers/commands/download.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Download parser code given log type."""
16 |
17 | import base64
18 | import os
19 | import pathlib
20 | import time
21 | from typing import AnyStr
22 |
23 | import click
24 |
25 | from common import api_utility
26 | from common import chronicle_auth
27 | from common import exception_handler
28 | from common import options
29 | from common.constants import key_constants as common_constants
30 | from common.constants import status
31 | from parsers import url
32 | from parsers.constants import key_constants
33 | from parsers.constants import path_constants
34 |
35 |
36 | @click.command(
37 | name="download", help="Download parser code given config ID or log type")
38 | @options.env_option
39 | @options.region_option
40 | @options.verbose_option
41 | @options.credential_file_option
42 | @exception_handler.catch_exception()
43 | def download(credential_file: AnyStr, verbose: bool, region: str,
44 | env: str) -> None:
45 | """Download parser code given log type or config ID.
46 |
47 | Args:
48 | credential_file (AnyStr): Path of Service Account JSON.
49 | verbose (bool): Option for printing verbose output to console.
50 | region (str): Option for selecting regions. Available options - US, EUROPE,
51 | ASIA_SOUTHEAST1.
52 | env (str): Option for selecting environment. Available options - prod, test.
53 |
54 | Raises:
55 | OSError: Failed to read the given file, e.g. not found, no read access
56 | (https://docs.python.org/library/exceptions.html#os-exceptions).
57 | ValueError: Invalid file contents.
58 | KeyError: Required key is not present in dictionary.
59 | TypeError: If response data is not JSON.
60 | """
61 | click.echo(
62 | "Note: If you want to download parser by log type then skip the config ID."
63 | )
64 | config_id = click.prompt("Enter config ID", show_default=False, default="")
65 |
66 | http_client = chronicle_auth.initialize_http_session(credential_file)
67 | method = "GET"
68 |
69 | if config_id:
70 | # Get the parser from config id
71 | download_parser_url = f'{url.get_url(region, "list", env)}/{config_id}'
72 | response = http_client.request(
73 | method, download_parser_url, timeout=url.HTTP_REQUEST_TIMEOUT_IN_SECS)
74 | download_parser_response = api_utility.check_content_type(response.text)
75 | else:
76 | # Get the parser of `log_type` from list of parsers
77 | log_type = click.prompt("Enter Log Type", show_default=False, default="")
78 | if not log_type:
79 | click.echo("Please enter log type or config ID.")
80 | return
81 | download_parser_url = url.get_url(region, "list", env)
82 | response = http_client.request(
83 | method, download_parser_url, timeout=url.HTTP_REQUEST_TIMEOUT_IN_SECS)
84 | download_parser_response = api_utility.check_content_type(response.text)
85 | if key_constants.KEY_CBN_PARSER not in download_parser_response:
86 | click.echo("No CBN parsers currently configured.")
87 | return
88 | found = False
89 | for p in download_parser_response[key_constants.KEY_CBN_PARSER]:
90 | if p[common_constants.KEY_LOG_TYPE] == log_type:
91 | found = True
92 | download_parser_response = p
93 | break
94 | if not found:
95 | click.echo(f"Parser for log type {log_type} not found.")
96 | return
97 |
98 | click.echo("Downloading parser...")
99 |
100 | if response.status_code != status.STATUS_OK:
101 | click.echo(
102 | f"Error while downloading parser:\nResponse Code: {response.status_code}"
103 | f"\nError: {download_parser_response[common_constants.KEY_ERROR][common_constants.KEY_MESSAGE]}"
104 | )
105 | return
106 |
107 | decoded_config = base64.b64decode(
108 | download_parser_response[key_constants.KEY_CONFIG])
109 | decoded_config = decoded_config.decode("utf-8")
110 | timestr = time.strftime("%Y%m%d%H%M%S")
111 | filename = download_parser_response[
112 | common_constants.KEY_LOG_TYPE] + "_" + timestr + ".conf"
113 | sample_dir = pathlib.Path(path_constants.PARSER_DATA_DIR)
114 | sample_dir.mkdir(parents=True, exist_ok=True)
115 | filepath = os.path.join(sample_dir, filename)
116 | click.echo(f"Writing parser to: {filepath}")
117 | with open(filepath, "w") as f:
118 | f.write(decoded_config)
119 |
120 | if verbose:
121 | api_utility.print_request_details(download_parser_url, method, None,
122 | download_parser_response)
123 |
--------------------------------------------------------------------------------
/parsers/commands/generate.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Generate sample logs for a given log type."""
16 |
17 | import pathlib
18 | from typing import AnyStr
19 |
20 | import click
21 |
22 | from common import api_utility
23 | from common import chronicle_auth
24 | from common import exception_handler
25 | from common import options
26 | from common.constants import key_constants as common_constants
27 | from common.constants import status
28 | from parsers import parser_utility
29 | from parsers import url
30 | from parsers.constants import key_constants
31 | from parsers.constants import path_constants
32 |
33 |
34 | @click.command(
35 | name='generate', help='Generate sample logs for a given log type')
36 | @options.env_option
37 | @options.region_option
38 | @options.credential_file_option
39 | @exception_handler.catch_exception()
40 | def generate(credential_file: AnyStr, region: str, env: str) -> None:
41 | """Generates sample data for writing parsers.
42 |
43 | Args:
44 | credential_file (AnyStr): Path of Service Account JSON.
45 | region (str): Option for selecting regions. Available options - US, EUROPE,
46 | ASIA_SOUTHEAST1.
47 | env (str): Option for selecting environment. Available options - prod, test.
48 |
49 | Raises:
50 | OSError: Failed to read the given file, e.g. not found, no read access
51 | (https://docs.python.org/library/exceptions.html#os-exceptions).
52 | ValueError: Invalid file contents.
53 | KeyError: Required key is not present in dictionary.
54 | TypeError: If response data is not JSON.
55 | """
56 | sample_sizes = ['1', '10', '1000']
57 | sample_names = ['1', '10', '1k']
58 |
59 | start_date = click.prompt(
60 | 'Enter Start Date (Format: yyyy-mm-ddThh:mm:ssZ)',
61 | default='',
62 | show_default=False)
63 | end_date = click.prompt(
64 | 'Enter End Date (Format: yyyy-mm-ddThh:mm:ssZ)',
65 | default='',
66 | show_default=False)
67 | log_type = click.prompt('Enter Log Type', default='', show_default=False)
68 |
69 | if (not start_date) or (not end_date) or (not log_type):
70 | click.echo(
71 | 'Start Date, End Date and Log Type are required. Please enter value for the missing field(s).'
72 | )
73 | return
74 |
75 | # Verify directory structure exists or create it.
76 | sample_dir = pathlib.Path(
77 | f'{path_constants.PARSER_DATA_DIR}/{log_type.lower()}')
78 | sample_dir.mkdir(parents=True, exist_ok=True)
79 |
80 | # Generate sample data of given sizes.
81 | for i, size in enumerate(sample_sizes):
82 | outfile = f'{sample_dir}/{log_type.lower()}_{sample_names[i]}.log'
83 | click.echo('\nGenerating sample size: {}... '.format(sample_names[i]),)
84 | call_get_sample_logs(credential_file, region, env, log_type.upper(),
85 | start_date, end_date, int(size), outfile)
86 |
87 | click.echo(
88 | f'\nGenerated sample data ({log_type.upper()}); run this to go there:')
89 | click.echo(f'cd {sample_dir}')
90 |
91 |
92 | def call_get_sample_logs(credential_file: str, region: str, env: str,
93 | log_type: str, start_time: str, end_time: str,
94 | number_of_entries: int, file_path: str) -> None:
95 | """Calls get sample logs endpoint and writes response to file."""
96 | data = {
97 | key_constants.KEY_LOG_TYPE: log_type,
98 | key_constants.KEY_START_TIME: start_time,
99 | key_constants.KEY_END_TIME: end_time,
100 | key_constants.KEY_MAX_ENTRIES: number_of_entries,
101 | }
102 | get_sample_log_url = f"{url.get_url(region, 'generate', env)}"
103 |
104 | # Make the request.
105 | client = chronicle_auth.initialize_http_session(credential_file)
106 | response = client.request(
107 | 'POST', get_sample_log_url, data, headers=url.HTTP_REQUEST_HEADERS)
108 | sample_logs = api_utility.check_content_type(response.text)
109 | if response.status_code != status.STATUS_OK:
110 | click.echo(
111 | f'Error while fetching status for parser.\nResponse Code: {response.status_code}'
112 | f'\nError: {sample_logs[common_constants.KEY_ERROR][common_constants.KEY_MESSAGE]}'
113 | )
114 | return
115 |
116 | # Parse the response.
117 | sample_logs_data = sample_logs.get(key_constants.KEY_DATA, [])
118 | with open(file_path, 'w') as f:
119 | for sample_log in sample_logs_data:
120 | f.write(parser_utility.decode_log(sample_log))
121 | f.write('\n')
122 |
--------------------------------------------------------------------------------
/parsers/commands/generate_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Tests for generate."""
16 |
17 | import shutil
18 | from typing import Dict
19 | from unittest import mock
20 |
21 | from click.testing import CliRunner
22 |
23 | from parsers.commands.generate import generate
24 | from parsers.tests.fixtures import * # pylint: disable=wildcard-import
25 | from parsers.tests.fixtures import MockResponse
26 | from parsers.tests.fixtures import TEST_DATA_DIR
27 |
28 |
29 | runner = CliRunner()
30 |
31 |
32 | @mock.patch(
33 | "common.chronicle_auth.initialize_http_session"
34 | )
35 | @mock.patch(
36 | "parsers.commands.generate.click.prompt")
37 | @mock.patch(
38 | "parsers.constants.path_constants.PARSER_DATA_DIR",
39 | TEST_DATA_DIR)
40 | def test_generate_logs(mock_input: mock.MagicMock, mock_client: mock.MagicMock,
41 | generate_logs: Dict[str, str]) -> None:
42 | """Test case to check response for generate logs.
43 |
44 | Args:
45 | mock_input (mock.MagicMock): Mock prompt object.
46 | mock_client (mock.MagicMock): Mock object.
47 | generate_logs (Tuple): Test input data.
48 | """
49 | result = runner.invoke(generate)
50 | mock_input.side_effect = [
51 | "2020-08-17T10:00:00Z", "2022-08-23T10:00:00Z", "WINDOWS_DHCP"
52 | ]
53 |
54 | # Check for non-empty response.
55 | mock_client.return_value = mock.Mock()
56 | mock_client.return_value.request.side_effect = [
57 | generate_logs, generate_logs, generate_logs
58 | ]
59 | result = runner.invoke(generate)
60 | assert "Generating sample size: 1" in result.output
61 | assert "Generating sample size: 10" in result.output
62 | assert "Generating sample size: 1k" in result.output
63 |
64 | try:
65 | shutil.rmtree(TEST_DATA_DIR + "/chronicle_cli/parsers/windows_dhcp")
66 | except FileNotFoundError:
67 | pass
68 |
69 |
70 | @mock.patch(
71 | "common.chronicle_auth.initialize_http_session"
72 | )
73 | @mock.patch(
74 | "parsers.commands.generate.click.prompt")
75 | @mock.patch(
76 | "parsers.constants.path_constants.PARSER_DATA_DIR",
77 | TEST_DATA_DIR)
78 | def test_empty_generate_logs(mock_input: mock.MagicMock,
79 | mock_client: mock.MagicMock) -> None:
80 | """Test case to check empty response for generate logs.
81 |
82 | Args:
83 | mock_input (mock.MagicMock): Mock prompt object.
84 | mock_client (mock.MagicMock): Mock object.
85 | """
86 | result = runner.invoke(generate)
87 | mock_input.side_effect = [
88 | "2020-08-17T10:00:00Z", "2022-08-23T10:00:00Z", "WINDOWS_DHCP"
89 | ]
90 | mock_input.side_effect = [
91 | "2020-08-17T10:00:00Z", "2022-08-23T10:00:00Z", "WINDOWS_DHCP"
92 | ]
93 | # Check for empty response.
94 | mock_client.return_value.request.side_effect = [
95 | MockResponse(status_code=200, text="""{}"""),
96 | MockResponse(status_code=200, text="""{}"""),
97 | MockResponse(status_code=200, text="""{}""")
98 | ]
99 | result = runner.invoke(generate)
100 | assert "Generating sample size: 1" in result.output
101 | assert "Generating sample size: 10" in result.output
102 | assert "Generating sample size: 1k" in result.output
103 | assert "Generated sample data (WINDOWS_DHCP); run this to go there:" in result.output
104 | assert "cd" in result.output
105 |
106 | try:
107 | shutil.rmtree(TEST_DATA_DIR + "/chronicle_cli/parsers/windows_dhcp")
108 | except FileNotFoundError:
109 | pass
110 |
111 |
112 | def test_prompt_text() -> None:
113 | """Test case to check prompt text."""
114 | result = runner.invoke(generate)
115 | assert "Enter Start Date (Format: yyyy-mm-ddThh:mm:ssZ):" in result.output
116 | assert "Enter End Date (Format: yyyy-mm-ddThh:mm:ssZ):" in result.output
117 | assert "Enter Log Type:" in result.output
118 |
--------------------------------------------------------------------------------
/parsers/commands/list.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """List all parsers of a given customer."""
16 |
17 | import os
18 | from typing import AnyStr
19 |
20 | import click
21 |
22 | from common import api_utility
23 | from common import chronicle_auth
24 | from common import exception_handler
25 | from common import file_utility
26 | from common import options
27 | from common.constants import key_constants as common_constants
28 | from common.constants import status
29 | from parsers import parser_templates
30 | from parsers import url
31 | from parsers.constants import key_constants as parser_constants
32 |
33 |
34 | @click.command(name="list", help="List all parsers of a given customer")
35 | @click.option(
36 | "-f",
37 | "--file-format",
38 | type=click.Choice(["TXT", "JSON"], case_sensitive=False),
39 | default="TXT",
40 | help="Format of the file to be exported")
41 | @options.export_option
42 | @options.env_option
43 | @options.region_option
44 | @options.verbose_option
45 | @options.credential_file_option
46 | @exception_handler.catch_exception()
47 | def list_command(credential_file: AnyStr, verbose: bool, region: str, env: str,
48 | export: AnyStr, file_format: AnyStr) -> None:
49 | """List all parsers of a given customer.
50 |
51 | Args:
52 | credential_file (AnyStr): Path of Service Account JSON.
53 | verbose (bool): Option for printing verbose output to console.
54 | region (str): Option for selecting regions. Available options - US, EUROPE,
55 | ASIA_SOUTHEAST1.
56 | env (str): Option for selecting environment. Available options - prod, test.
57 | export (AnyStr): Path of file to export output of list command.
58 | file_format (AnyStr): Format of the content to be exported.
59 |
60 | Raises:
61 | OSError: Failed to read the given file, e.g. not found, no read access
62 | (https://docs.python.org/library/exceptions.html#os-exceptions).
63 | ValueError: Invalid file contents.
64 | KeyError: Required key is not present in dictionary.
65 | TypeError: If response data is not JSON.
66 | """
67 | click.echo("Fetching list of parsers...")
68 | list_parser_url = url.get_url(region, "list", env)
69 | client = chronicle_auth.initialize_http_session(credential_file)
70 | method = "GET"
71 | response = client.request(
72 | method, list_parser_url, timeout=url.HTTP_REQUEST_TIMEOUT_IN_SECS)
73 | parser_response = api_utility.check_content_type(response.text)
74 |
75 | if response.status_code != status.STATUS_OK:
76 | click.echo(
77 | f"Error while fetching list of parsers.\nResponse Code: {response.status_code}"
78 | f"\nError: {parser_response[common_constants.KEY_ERROR][common_constants.KEY_MESSAGE]}"
79 | )
80 | return
81 |
82 | if parser_constants.KEY_CBN_PARSER not in parser_response:
83 | click.echo("No CBN parsers currently configured.")
84 | return
85 |
86 | parser_details = ""
87 | for parser in parser_response[parser_constants.KEY_CBN_PARSER]:
88 | try:
89 | del parser[parser_constants.KEY_CONFIG]
90 | parser_details += parser_templates.parser_details_template.substitute(
91 | config_id=f"{parser[parser_constants.KEY_CONFIG_ID]}",
92 | log_type=f"{parser[common_constants.KEY_LOG_TYPE]}",
93 | state=f"{parser[parser_constants.KEY_STATE]}",
94 | sha256=f"{parser[parser_constants.KEY_SHA256]}",
95 | author=f'{parser.get(parser_constants.KEY_AUTHOR, "-")}',
96 | submit_time=f"{parser[parser_constants.KEY_SUBMIT_TIME]}",
97 | last_live_time=f'{parser.get(parser_constants.KEY_LAST_LIVE_TIME, "-")}',
98 | state_last_changed_time=f'{parser.get(parser_constants.KEY_STATE_LAST_CHANGED_TIME, "-")}',
99 | )
100 | except KeyError as e:
101 | parser_details += f"\nKey {str(e)} not found in the response."
102 | except Exception as e: # pylint: disable=broad-except
103 | parser_details += f"\nFailed with exception: str({e})"
104 | parser_details += f'\n\n{"=" * 60}\n'
105 |
106 | click.echo(parser_details)
107 |
108 | if export:
109 | export_path = os.path.abspath(export) + f".{file_format.lower()}"
110 | if file_format == file_utility.FILE_FORMAT_JSON:
111 | file_utility.export_json(export_path, parser_response)
112 | else:
113 | file_utility.export_txt(export_path, parser_details)
114 | click.echo(f"\nParser details exported successfully to: {export_path}")
115 |
116 | if verbose:
117 | api_utility.print_request_details(list_parser_url, method, None,
118 | parser_response)
119 |
--------------------------------------------------------------------------------
/parsers/commands/run.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Run the parser against given logs."""
16 |
17 | import base64
18 | import time
19 | from typing import AnyStr
20 | import urllib
21 |
22 | import click
23 |
24 | from common import api_utility
25 | from common import chronicle_auth
26 | from common import exception_handler
27 | from common import file_utility
28 | from common import options
29 | from common.constants import key_constants as common_constants
30 | from common.constants import status
31 | from parsers import url
32 | from parsers.constants import key_constants as parser_constants
33 |
34 |
35 | @click.command(name='run', help='Run the parser against given logs')
36 | @options.env_option
37 | @options.region_option
38 | @options.verbose_option
39 | @options.credential_file_option
40 | @exception_handler.catch_exception()
41 | def run(credential_file: AnyStr, verbose: bool, region: str, env: str) -> None:
42 | """Run the parser against given logs.
43 |
44 | Args:
45 | credential_file (AnyStr): Path of Service Account JSON.
46 | verbose (bool): Option for printing verbose output to console.
47 | region (str): Option for selecting regions. Available options - US, EUROPE,
48 | ASIA_SOUTHEAST1.
49 | env (str): Option for selecting environment. Available options - prod, test.
50 |
51 | Raises:
52 | OSError: Failed to read the given file, e.g. not found, no read access
53 | (https://docs.python.org/library/exceptions.html#os-exceptions).
54 | ValueError: Invalid file contents.
55 | KeyError: Required key is not present in dictionary.
56 | TypeError: If response data is not JSON.
57 | """
58 | conf_file_path = click.prompt('Enter path for conf file')
59 | log_file_path = click.prompt('Enter path for log file')
60 | click.echo('Running Validation...')
61 | start_time = time.time()
62 |
63 | config_data = file_utility.read_file(conf_file_path)
64 | log_data = file_utility.read_file(log_file_path)
65 |
66 | data = urllib.parse.urlencode({
67 | parser_constants.KEY_CONFIG: base64.urlsafe_b64encode(config_data),
68 | parser_constants.KEY_LOGS: base64.urlsafe_b64encode(log_data)
69 | })
70 |
71 | run_parser_url = url.get_url(region, 'run', env)
72 | method = 'POST'
73 | client = chronicle_auth.initialize_http_session(credential_file)
74 |
75 | response = client.request(
76 | method,
77 | run_parser_url,
78 | data=data,
79 | headers=url.HTTP_REQUEST_HEADERS,
80 | timeout=url.HTTP_REQUEST_TIMEOUT_IN_SECS)
81 | parser_response = api_utility.check_content_type(response.text)
82 |
83 | if response.status_code != status.STATUS_OK:
84 | click.echo(
85 | f'Error while running validation.\nResponse Code: {response.status_code}'
86 | f'\nError: {parser_response[common_constants.KEY_ERROR][common_constants.KEY_MESSAGE]}'
87 | )
88 | return
89 |
90 | for result in parser_response.get(parser_constants.KEY_RESULT, []):
91 | click.echo(result)
92 |
93 | for err in parser_response.get(common_constants.KEY_ERRORS, []):
94 | click.echo(err[parser_constants.KEY_ERROR_MSG])
95 | click.echo(err[parser_constants.KEY_LOG_ENTRY])
96 |
97 | time_elapsed = time.time() - start_time
98 | click.echo('Runtime: {:.5}s'.format(time_elapsed))
99 |
100 | if verbose:
101 | api_utility.print_request_details(run_parser_url, method, None,
102 | parser_response)
103 |
--------------------------------------------------------------------------------
/parsers/commands/run_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Tests for run.py."""
16 |
17 | from unittest import mock
18 |
19 | from click.testing import CliRunner
20 |
21 | from mock_test_utility import MockResponse
22 | from parsers.commands.run import run
23 | from parsers.tests.fixtures import * # pylint: disable=wildcard-import
24 |
25 | runner = CliRunner()
26 |
27 |
28 | @mock.patch(
29 | "common.chronicle_auth.initialize_http_session"
30 | )
31 | @mock.patch("parsers.url.get_url")
32 | @mock.patch("common.file_utility.read_file")
33 | @mock.patch(
34 | "parsers.commands.run.click.prompt")
35 | def test_run_command(mock_input: mock.MagicMock, mock_read_file: mock.MagicMock,
36 | mock_url: mock.MagicMock, mock_client: mock.MagicMock,
37 | test_run_validation_data: MockResponse) -> None:
38 | """Test case to check response for run parsers.
39 |
40 | Args:
41 | mock_input: Mock object
42 | mock_read_file: Mock object
43 | mock_url (mock.MagicMock): Mock object
44 | mock_client (mock.MagicMock): Mock object
45 | test_run_validation_data (Tuple): Test input data
46 | """
47 | mock_input.side_effect = ["path1", "path2"]
48 | mock_read_file.side_effect = [b"conf", b"log"]
49 | mock_url.return_value = "test_url"
50 | mock_client.return_value = mock.Mock()
51 | mock_client.return_value.request.side_effect = [test_run_validation_data]
52 | result = runner.invoke(run)
53 | assert """Running Validation...
54 | result 1
55 | result 2
56 | """ in result.output
57 |
58 |
59 | @mock.patch(
60 | "common.chronicle_auth.initialize_http_session"
61 | )
62 | @mock.patch("parsers.url.get_url")
63 | @mock.patch("common.file_utility.read_file")
64 | @mock.patch("parsers.commands.run.time.time")
65 | @mock.patch(
66 | "parsers.commands.run.click.prompt")
67 | def test_run_command_error(
68 | mock_input: mock.MagicMock, mock_time: mock.MagicMock,
69 | mock_read_file: mock.MagicMock, mock_url: mock.MagicMock,
70 | mock_client: mock.MagicMock,
71 | test_run_validation_error_data: MockResponse) -> None:
72 | """Test case to check response for run parsers in case of error.
73 |
74 | Args:
75 | mock_input: Mock object
76 | mock_time: Mock object
77 | mock_read_file: Mock object
78 | mock_url (mock.MagicMock): Mock object
79 | mock_client (mock.MagicMock): Mock object
80 | test_run_validation_error_data (Tuple): Test input data
81 | """
82 | mock_input.side_effect = ["path1", "path2"]
83 | mock_time.side_effect = [2.1, 5.2]
84 | mock_read_file.side_effect = [b"conf", b"log"]
85 | mock_url.return_value = "test_url"
86 | mock_client.return_value = mock.Mock()
87 | mock_client.return_value.request.side_effect = [
88 | test_run_validation_error_data
89 | ]
90 | result = runner.invoke(run)
91 | assert """Running Validation...
92 | test error
93 | sample log
94 | Runtime: 3.1s
95 | """ in result.output
96 |
97 |
98 | @mock.patch(
99 | "common.chronicle_auth.initialize_http_session"
100 | )
101 | @mock.patch("parsers.url.get_url")
102 | @mock.patch("common.file_utility.read_file")
103 | @mock.patch(
104 | "parsers.commands.run.click.prompt")
105 | def test_run_command_500(mock_input: mock.MagicMock,
106 | mock_read_file: mock.MagicMock,
107 | mock_url: mock.MagicMock, mock_client: mock.MagicMock,
108 | test_500_resp: MockResponse) -> None:
109 | """Test case to check response for run parsers in case of 500 response code.
110 |
111 | Args:
112 | mock_input: Mock object
113 | mock_read_file: Mock object
114 | mock_url (mock.MagicMock): Mock object
115 | mock_client (mock.MagicMock): Mock object
116 | test_500_resp (Tuple): Test input data
117 | """
118 | mock_input.side_effect = ["path1", "path2"]
119 | mock_read_file.side_effect = [b"conf", b"log"]
120 | mock_url.return_value = "test_url"
121 | mock_client.return_value = mock.Mock()
122 | mock_client.return_value.request.side_effect = [test_500_resp]
123 | result = runner.invoke(run)
124 | assert """Running Validation...
125 | Error while running validation.
126 | Response Code: 500
127 | Error: test error
128 | """ in result.output
129 |
--------------------------------------------------------------------------------
/parsers/commands/status.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Get status of a submitted parser."""
16 |
17 | from typing import AnyStr
18 |
19 | import click
20 |
21 | from common import api_utility
22 | from common import chronicle_auth
23 | from common import exception_handler
24 | from common import options
25 | from common.constants import key_constants as common_constants
26 | from common.constants import status
27 | from parsers import parser_templates
28 | from parsers import url
29 | from parsers.constants import key_constants as parser_constants
30 |
31 |
32 | @click.command(name="status", help="Get status of a submitted parser")
33 | @options.env_option
34 | @options.verbose_option
35 | @options.region_option
36 | @options.credential_file_option
37 | @exception_handler.catch_exception()
38 | def status_command(credential_file: AnyStr, verbose: bool, region: str,
39 | env: str) -> None:
40 | """Get status of a submitted parser.
41 |
42 | Args:
43 | credential_file (AnyStr): Path of Service Account JSON.
44 | verbose (bool): Option for printing verbose output to console.
45 | region (str): Option for selecting regions. Available options - US, EUROPE,
46 | ASIA_SOUTHEAST1.
47 | env (str): Option for selecting environment. Available options - prod, test.
48 |
49 | Raises:
50 | OSError: Failed to read the given file, e.g. not found, no read access
51 | (https://docs.python.org/library/exceptions.html#os-exceptions).
52 | ValueError: Invalid file contents.
53 | KeyError: Required key is not present in dictionary.
54 | TypeError: If response data is not JSON.
55 | """
56 | config_id = click.prompt("Enter Config ID", default="", show_default=False)
57 |
58 | if not config_id:
59 | click.echo("\nPlease enter config id.")
60 | return
61 |
62 | click.echo("\nGetting parser...")
63 |
64 | # Make the request.
65 | get_parser_status_url = f"{url.get_url(region, 'status', env)}/{config_id}"
66 | method = "GET"
67 | client = chronicle_auth.initialize_http_session(credential_file)
68 | response = client.request(
69 | method, get_parser_status_url, timeout=url.HTTP_REQUEST_TIMEOUT_IN_SECS)
70 | parser = api_utility.check_content_type(response.text)
71 |
72 | if response.status_code != status.STATUS_OK:
73 | click.echo(
74 | f"Error while fetching status for parser.\nResponse Code: {response.status_code}"
75 | f"\nError: {parser[common_constants.KEY_ERROR][common_constants.KEY_MESSAGE]}"
76 | )
77 | return
78 |
79 | parser_details = parser_templates.parser_details_template.substitute(
80 | config_id=f"{parser[parser_constants.KEY_CONFIG_ID]}",
81 | log_type=f"{parser[common_constants.KEY_LOG_TYPE]}",
82 | state=f"{parser[parser_constants.KEY_STATE]}",
83 | sha256=f"{parser[parser_constants.KEY_SHA256]}",
84 | author=f"{parser[parser_constants.KEY_AUTHOR]}",
85 | submit_time=f"{parser[parser_constants.KEY_SUBMIT_TIME]}",
86 | last_live_time=f'{parser.get(parser_constants.KEY_LAST_LIVE_TIME, "-")}',
87 | state_last_changed_time=f"{parser[parser_constants.KEY_STATE_LAST_CHANGED_TIME]}"
88 | )
89 | click.echo(parser_details)
90 |
91 | if verbose:
92 | api_utility.print_request_details(get_parser_status_url, method, None,
93 | parser)
94 |
--------------------------------------------------------------------------------
/parsers/commands/status_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Tests for status."""
16 |
17 | from unittest import mock
18 |
19 | from click.testing import CliRunner
20 |
21 | from common import uri
22 | from mock_test_utility import MockResponse
23 | from parsers import url
24 | from parsers.commands.status import status_command
25 | from parsers.tests.fixtures import * # pylint: disable=wildcard-import
26 |
27 |
28 | runner = CliRunner()
29 |
30 |
31 | @mock.patch(
32 | "common.chronicle_auth.initialize_http_session"
33 | )
34 | @mock.patch(
35 | "parsers.commands.status.click.prompt")
36 | def test_status_parser(mock_input: mock.MagicMock, mock_client: mock.MagicMock,
37 | status_parser: MockResponse) -> None:
38 | """Test case to check response for status of parser.
39 |
40 | Args:
41 | mock_input (mock.MagicMock): Mock prompt object.
42 | mock_client (mock.MagicMock): Mock object.
43 | status_parser (Tuple): Test input data.
44 | """
45 | mock_input.side_effect = ["test_config_id"]
46 | # Check for non-empty response.
47 | mock_client.return_value = mock.Mock()
48 | mock_client.return_value.request.side_effect = [status_parser]
49 | result = runner.invoke(status_command)
50 | assert """Getting parser...
51 |
52 | Parser Details:
53 | Config ID: test_config_id
54 | Log type: TEST_LOG_TYPE
55 | State: LIVE
56 | SHA256: test_sha256
57 | Author: test_user
58 | Submit Time: 2022-04-01T08:08:44.217797Z
59 | State Last Changed Time: 2022-04-01T08:08:44.217797Z
60 | Last Live Time: 2022-04-01T08:08:44.217797Z
61 | """ in result.output
62 | mock_client.return_value.request.assert_called_once_with(
63 | "GET",
64 | f"{uri.BASE_URL}/v1/tools/cbnParsers/test_config_id",
65 | timeout=url.HTTP_REQUEST_TIMEOUT_IN_SECS)
66 |
67 |
68 | @mock.patch(
69 | "parsers.commands.status.click.prompt")
70 | def test_prompt_required(mock_input: mock.MagicMock) -> None:
71 | """Test case to check prompt text.
72 |
73 | Args:
74 | mock_input (mock.MagicMock): Mock prompt object.
75 | """
76 | mock_input.side_effect = [""]
77 | result = runner.invoke(status_command)
78 | assert "Please enter config id." in result.output
79 |
80 |
81 | @mock.patch(
82 | "common.chronicle_auth.initialize_http_session"
83 | )
84 | @mock.patch("parsers.url.get_url")
85 | @mock.patch(
86 | "parsers.commands.status.click.prompt")
87 | def test_status_parser_400(mock_input: mock.MagicMock, mock_url: mock.MagicMock,
88 | mock_client: mock.MagicMock,
89 | test_400_resp_status_command: MockResponse) -> None:
90 | """Test case to check response for status of parser for 500 response code.
91 |
92 | Args:
93 | mock_input (mock.MagicMock): Mock prompt object.
94 | mock_url (mock.MagicMock): Mock object.
95 | mock_client (mock.MagicMock): Mock object.
96 | test_400_resp_status_command (Tuple): Test response data.
97 | """
98 | mock_input.side_effect = ["test_config_id"]
99 | mock_url.return_value = f"{uri.BASE_URL}/tools/cbnParsers"
100 | mock_client.return_value = mock.Mock()
101 | mock_client.return_value.request.side_effect = [test_400_resp_status_command]
102 | result = runner.invoke(status_command)
103 | assert """Getting parser...
104 | Error while fetching status for parser.
105 | Response Code: 400
106 | Error: Invalid ID.
107 | """ in result.output
108 |
109 |
110 | @mock.patch(
111 | "common.chronicle_auth.initialize_http_session"
112 | )
113 | @mock.patch("parsers.url.get_url")
114 | @mock.patch(
115 | "parsers.commands.status.click.prompt")
116 | def test_status_parser_500(mock_input: mock.MagicMock, mock_url: mock.MagicMock,
117 | mock_client: mock.MagicMock,
118 | test_500_resp_status_command: MockResponse) -> None:
119 | """Test case to check response for status of parser for 500 response code.
120 |
121 | Args:
122 | mock_input (mock.MagicMock): Mock prompt object.
123 | mock_url (mock.MagicMock): Mock object.
124 | mock_client (mock.MagicMock): Mock object.
125 | test_500_resp_status_command (Tuple): Test response data.
126 | """
127 | mock_input.side_effect = ["test_config_id"]
128 | mock_url.return_value = f"{uri.BASE_URL}/tools/cbnParsers"
129 | mock_client.return_value = mock.Mock()
130 | mock_client.return_value.request.side_effect = [test_500_resp_status_command]
131 | result = runner.invoke(status_command)
132 | assert """Getting parser...
133 | Error while fetching status for parser.
134 | Response Code: 500
135 | Error: could not get CBN parser.
136 | """ in result.output
137 |
138 |
139 | def test_prompt_text() -> None:
140 | """Test case to check prompt text."""
141 | result = runner.invoke(status_command)
142 | assert "Enter Config ID:" in result.output
143 |
--------------------------------------------------------------------------------
/parsers/commands/submit.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Submit a new parser."""
16 |
17 | import base64
18 | import os
19 | from typing import AnyStr
20 | import urllib
21 |
22 | import click
23 |
24 | from common import api_utility
25 | from common import chronicle_auth
26 | from common import exception_handler
27 | from common import options
28 | from common.constants import key_constants as common_constants
29 | from common.constants import status
30 | from parsers import parser_templates
31 | from parsers import url
32 | from parsers.constants import key_constants as parser_constants
33 |
34 |
35 | @click.command(name='submit', help='Submit a new parser')
36 | @options.env_option
37 | @options.region_option
38 | @options.verbose_option
39 | @options.credential_file_option
40 | @exception_handler.catch_exception()
41 | def submit(credential_file: AnyStr,
42 | verbose: bool,
43 | region: str,
44 | env: str) -> None:
45 | """Submit a new parser.
46 |
47 | Args:
48 | credential_file (AnyStr): Path of Service Account JSON.
49 | verbose (bool): Option for printing verbose output to console.
50 | region (str): Option for selecting regions. Available options - US, EUROPE,
51 | ASIA_SOUTHEAST1.
52 | env (str): Option for selecting environment. Available options - prod, test.
53 |
54 | Raises:
55 | OSError: Failed to read the given file, e.g. not found, no read access
56 | (https://docs.python.org/library/exceptions.html#os-exceptions).
57 | ValueError: Invalid file contents.
58 | KeyError: Required key is not present in dictionary.
59 | TypeError: If response data is not JSON.
60 | """
61 | log_type = click.prompt('Enter Log type', show_default=False, default='')
62 | conf_file = click.prompt(
63 | 'Enter Config file path', show_default=False, default='')
64 | author = click.prompt('Enter author', show_default=False, default='')
65 | skip_validation_on_no_logs = click.prompt(
66 | 'Skip validation if no logs found',
67 | type=bool,
68 | show_default=True,
69 | default=False)
70 |
71 | if (not log_type) or (not conf_file) or (not author):
72 | click.echo('Log Type, Config file path and Author fields are required. '
73 | 'Please enter value for the missing field(s).')
74 | return
75 |
76 | if not os.path.exists(conf_file):
77 | click.echo(
78 | f'{conf_file} does not exist. Please enter valid config file path.')
79 | return
80 |
81 | with open(conf_file, 'rb') as config_file:
82 | config_data = config_file.read()
83 |
84 | skip_validation_key = parser_constants.KEY_SKIP_VALIDATION_ON_NO_LOGS
85 | data = {
86 | parser_constants.KEY_LOG_TYPE: log_type,
87 | parser_constants.KEY_CONFIG: base64.urlsafe_b64encode(config_data),
88 | parser_constants.KEY_AUTHOR: author,
89 | skip_validation_key: skip_validation_on_no_logs,
90 | }
91 |
92 | click.echo('Submitting parser...')
93 |
94 | request_body = urllib.parse.urlencode(data)
95 | submit_parser_url = url.get_url(region, 'list', env)
96 | client = chronicle_auth.initialize_http_session(credential_file)
97 | method = 'POST'
98 | response = client.request(
99 | method,
100 | submit_parser_url,
101 | request_body,
102 | headers=url.HTTP_REQUEST_HEADERS,
103 | timeout=url.HTTP_REQUEST_TIMEOUT_IN_SECS)
104 | parser = api_utility.check_content_type(response.text)
105 |
106 | if response.status_code != status.STATUS_OK:
107 | click.echo(
108 | f'Error while submitting parser:\nResponse Code: {response.status_code}'
109 | f'\nError: {parser[common_constants.KEY_ERROR][common_constants.KEY_MESSAGE]}'
110 | )
111 | return
112 |
113 | del parser[parser_constants.KEY_CONFIG]
114 |
115 | parser_details = parser_templates.submitted_parser_details_template.substitute(
116 | config_id=f'{parser[parser_constants.KEY_CONFIG_ID]}',
117 | log_type=f'{parser[common_constants.KEY_LOG_TYPE]}',
118 | submit_time=f'{parser[parser_constants.KEY_SUBMIT_TIME]}',
119 | state_last_changed_time=f'{parser[parser_constants.KEY_STATE_LAST_CHANGED_TIME]}',
120 | state=f'{parser[parser_constants.KEY_STATE]}',
121 | sha256=f'{parser[parser_constants.KEY_SHA256]}',
122 | author=f'{parser[parser_constants.KEY_AUTHOR]}')
123 | click.echo(parser_details)
124 |
125 | click.echo(
126 | '\nParser submitted successfully. To get status of the parser, run this '
127 | f'command using following Config ID - {parser[parser_constants.KEY_CONFIG_ID]}:'
128 | )
129 | click.echo('`chronicle_cli parsers status`')
130 |
131 | if verbose:
132 | api_utility.print_request_details(submit_parser_url, method, None, parser)
133 |
--------------------------------------------------------------------------------
/parsers/constants/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 |
--------------------------------------------------------------------------------
/parsers/constants/key_constants.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Common constants to be used for parser tool."""
16 |
17 |
18 | KEY_AUTHOR = 'author'
19 | KEY_AVG_PARSE_DURATION = 'avgParseDuration'
20 | KEY_CATEGORY = 'category'
21 | KEY_CBN = 'cbn'
22 | KEY_CBN_PARSER = 'cbnParsers'
23 | KEY_CBN_SNIPPET = 'cbnSnippet'
24 | KEY_CHANGELOGS = 'changelogs'
25 | KEY_CONFIG = 'config'
26 | KEY_CONFIG_ID = 'configId'
27 | KEY_CREATE_TIME = 'createTime'
28 | KEY_CREATOR = 'creator'
29 | KEY_CUSTOMER = 'customer'
30 | KEY_DATA = 'data'
31 | KEY_DROP_TAG_COUNTS = 'dropTagCounts'
32 | KEY_END_TIME = 'end_time'
33 | KEY_ENTRIES = 'entries'
34 | KEY_ERROR = 'error'
35 | KEY_ERROR_ID = 'errorId'
36 | KEY_ERROR_MESSAGE = 'errorMsg'
37 | KEY_ERROR_MSG = 'errorMsg'
38 | KEY_ERROR_TIME = 'errorTime'
39 | KEY_ERRORS = 'errors'
40 | KEY_EVENT_CATEGORY_COUNTS = 'eventCategoryCounts'
41 | KEY_EVENT_COUNT = 'eventCount'
42 | KEY_EXTENSION_VALIDATION_REPORT = 'extensionValidationReport'
43 | KEY_FAILED_LOG_COUNT = 'failedLogCount'
44 | KEY_FIELD_EXTRACTORS = 'fieldExtractors'
45 | KEY_GENERIC_EVENT_COUNT = 'genericEventCount'
46 | KEY_GENERIC_EVENT_PERCENTAGE = 'genericEventPercentage'
47 | KEY_INPUT_CBN_SNIPPET = 'cbn_snippet'
48 | KEY_INSTANCE = 'instance'
49 | KEY_INSTANCES = 'instances'
50 | KEY_INVALID_CONFIG_ERROR = 'invalidConfigError'
51 | KEY_INVALID_LOG_COUNT = 'invalidLogCount'
52 | KEY_LAST_LIVE_TIME = 'lastLiveTime'
53 | KEY_LOCATION = 'location'
54 | KEY_LOCATIONS = 'locations'
55 | KEY_LOG = 'log'
56 | KEY_LOG_DATA = 'logData'
57 | KEY_LOG_ENTRY = 'logEntry'
58 | KEY_LOG_ENTRY_COUNT = 'logEntryCount'
59 | KEY_LOG_TYPE = 'log_type'
60 | KEY_LOGS = 'logs'
61 | KEY_LOGTYPE = 'logType'
62 | KEY_LOGTYPES = 'logTypes'
63 | KEY_LOW_CODE = 'lowCode'
64 | KEY_MAX_ENTRIES = 'max_entries'
65 | KEY_MAX_PARSE_DURATION = 'maxParseDuration'
66 | KEY_MESSAGE = 'message'
67 | KEY_NAME = 'name'
68 | KEY_NORMALIZATION_PERCENTAGE = 'normalizationPercentage'
69 | KEY_ON_ERROR_COUNT = 'onErrorCount'
70 | KEY_PARSED_EVENTS = 'parsedEvents'
71 | KEY_PARSER = 'parser'
72 | KEY_PARSER_EXTENSION = 'parserExtension'
73 | KEY_PARSER_EXTENSION_ID = 'parserExtensionID'
74 | KEY_PARSER_EXTENSIONS = 'parserExtensions'
75 | KEY_PARSER_ID = 'parserID'
76 | KEY_PARSERS = 'parsers'
77 | KEY_PARSING_ERRORS = 'parsingErrors'
78 | KEY_PARSING_ERRORS = 'parsingErrors'
79 | KEY_PARSING_ERRORS = 'parsingErrors'
80 | KEY_PREDICTIONS = 'predictions'
81 | KEY_PROJECT = 'project'
82 | KEY_PROJECTS = 'projects'
83 | KEY_RELEASE_STAGE = 'releaseStage'
84 | KEY_RESULT = 'result'
85 | KEY_RUN_PARSER_RESULTS = 'runParserResults'
86 | KEY_SHA256 = 'sha256'
87 | KEY_SKIP_VALIDATION_ON_NO_LOGS = 'skipValidationOnNoLogs'
88 | KEY_SCORE = 'score'
89 | KEY_SOURCE = 'source'
90 | KEY_START_TIME = 'start_time'
91 | KEY_STATE = 'state'
92 | KEY_STATE_LAST_CHANGED_TIME = 'stateLastChangedTime'
93 | KEY_STATEDUMP_ALLOWED = 'statedump_allowed'
94 | KEY_STATEDUMP_RESULT = 'statedumpResult'
95 | KEY_STATEDUMP_RESULTS = 'statedumpResults'
96 | KEY_STATS = 'stats'
97 | KEY_SUBMIT_TIME = 'submitTime'
98 | KEY_SUCCESSFULLY_NORMALIZED_LOG_COUNT = 'successfullyNormalizedLogCount'
99 | KEY_TYPE = 'type'
100 | KEY_VALIDATION_ERRORS = 'validationErrors'
101 | KEY_VALIDATED_ON_EMPTY_LOGS = 'validatedOnEmptyLogs'
102 | KEY_VALIDATION_REPORT = 'validationReport'
103 | KEY_VALIDATION_REPORT_ID = 'validationReportID'
104 | KEY_VALIDATION_REPORTS = 'validationReports'
105 | KEY_VALIDATION_STAGE = 'validationStage'
106 | KEY_VERDICT = 'verdict'
107 |
--------------------------------------------------------------------------------
/parsers/constants/path_constants.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Path constants to be used for parser tool."""
16 | import pathlib
17 |
18 | PARSER_DATA_DIR = f'{pathlib.Path.home()}/chronicle_cli/parsers'
19 |
--------------------------------------------------------------------------------
/parsers/parser_templates.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Templates for printing output to console."""
16 |
17 | import string
18 |
19 | parser_details_template = string.Template("""\
20 |
21 | Parser Details:
22 | Config ID: ${config_id}
23 | Log type: ${log_type}
24 | State: ${state}
25 | SHA256: ${sha256}
26 | Author: ${author}
27 | Submit Time: ${submit_time}
28 | State Last Changed Time: ${state_last_changed_time}
29 | Last Live Time: ${last_live_time}""")
30 |
31 | errors_details_template = string.Template("""\
32 |
33 | Error Details:
34 | Error ID: ${error_id}
35 | Config ID: ${config_id}
36 | Log type: ${log_type}
37 | Error Time: ${error_time}
38 | Error Category: ${category}
39 | Error Message: ${error_msg}
40 | ${logs}""")
41 |
42 | submitted_parser_details_template = string.Template("""\
43 |
44 | Submitted Parser Details:
45 | Config ID: ${config_id}
46 | Log type: ${log_type}
47 | State: ${state}
48 | SHA256: ${sha256}
49 | Author: ${author}
50 | Submit Time: ${submit_time}
51 | State Last Changed Time: ${state_last_changed_time}
52 | """)
53 |
54 | parser_history_template = string.Template("""\
55 |
56 | Parser History:
57 | Config ID: ${config_id}
58 | Log type: ${log_type}
59 | State: ${state}
60 | SHA256: ${sha256}
61 | Author: ${author}
62 | Submit Time: ${submit_time}
63 | State Last Changed Time: ${state_last_changed_time}\
64 | ${last_live_time}${validationErrors}""")
65 |
66 | parserextension_details_template = string.Template("""\
67 |
68 | ParserExtension Details:
69 | ParserExtension ID: ${parserextension_id}
70 | Log type: ${log_type}
71 | State: ${state}
72 | Validation Report ID: ${validation_report_id}
73 | Create Time: ${create_time}
74 | State Last Changed Time: ${state_last_changed_time}
75 | Last Live Time: ${last_live_time}""")
76 |
77 | parserv2_details_template = string.Template("""\
78 |
79 | Parser Details:
80 | Parser ID: ${parser_id}
81 | Log type: ${log_type}
82 | State: ${state}
83 | Type: ${type}
84 | Author: ${author}
85 | Validation Report ID: ${validation_report_id}
86 | Create Time: ${create_time}""")
87 |
88 | parsing_errors_details_template = string.Template("""\
89 |
90 | Log: ${log}
91 | Error: ${error}""")
92 |
93 | validation_report_template = string.Template("""\
94 |
95 | Validation Report:
96 | Verdict: ${verdict}
97 | Stats:
98 | LogEntry Count: ${log_entry_count}
99 | Successfully Normalized Log Count: ${successfully_normalized_log_count}
100 | Failed Log Count: ${failed_log_count}
101 | Invalid Log Count: ${invalid_log_count}
102 | On Error Count: ${on_error_count}
103 | Event Count: ${event_count}
104 | Generic Event Count: ${generic_event_count}
105 | Event Category:
106 | ${event_category_count}
107 | Drop Tag:
108 | ${drop_tag_count}
109 | Max Parse Duration: ${max_parse_duration}
110 | Avg Parse Duration: ${avg_parse_duration}
111 | Normalization percent: ${normalization_percentage}
112 | Generic Event percent: ${generic_event_percentage}
113 | Errors: ${errors}""")
114 |
--------------------------------------------------------------------------------
/parsers/parser_utility.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Parser utility functions."""
16 |
17 | import base64
18 |
19 | from typing import Dict
20 |
21 |
22 | def decode_log(log: str) -> str:
23 | """Decode the log data from the response.
24 |
25 | Args:
26 | log: Encoded log
27 |
28 | Returns:
29 | Decoded log
30 | """
31 | log_bytes = base64.b64decode(log)
32 | return log_bytes.decode(encoding='utf-8', errors='surrogateescape')
33 |
34 |
35 | def process_resource_name(name: str) -> Dict[str, str]:
36 | """Extract resource components from the resource name.
37 |
38 | Args:
39 | name (str): The resource name
40 |
41 | Returns:
42 | (Dict): Resource components
43 | """
44 | processed_fields = {}
45 | name_split = name.split('/')
46 | for i in range(0, len(name_split), 2):
47 | processed_fields[name_split[i]] = name_split[i + 1]
48 | return processed_fields
49 |
--------------------------------------------------------------------------------
/parsers/parser_utility_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Unit tests for parser_utility."""
16 |
17 | from parsers import parser_utility
18 | from parsers.constants import key_constants
19 |
20 |
21 | def test_decode_log() -> None:
22 | """Test decode log."""
23 | input_log = 'dGVzdF9sb2c='
24 | assert parser_utility.decode_log(input_log) == 'test_log'
25 |
26 |
27 | def test_process_resource_name_for_parser() -> None:
28 | """Test process resource name for parser."""
29 | parent = 'projects/test_project/locations/test_location/instances/test_instance/logTypes/test_log_type/parsers/test_parser_id'
30 | component = parser_utility.process_resource_name(parent)
31 | assert component == {
32 | key_constants.KEY_PROJECTS: 'test_project',
33 | key_constants.KEY_LOCATIONS: 'test_location',
34 | key_constants.KEY_INSTANCES: 'test_instance',
35 | key_constants.KEY_LOGTYPES: 'test_log_type',
36 | key_constants.KEY_PARSERS: 'test_parser_id'
37 | }
38 |
39 |
40 | def test_process_resource_name_for_parserextension() -> None:
41 | """Test process resource name for parserextension."""
42 | parent = 'projects/test_project/locations/test_location/instances/test_instance/logTypes/test_log_type/parserExtensions/test_parserextension_id'
43 | component = parser_utility.process_resource_name(parent)
44 | assert component == {
45 | key_constants.KEY_PROJECTS: 'test_project',
46 | key_constants.KEY_LOCATIONS: 'test_location',
47 | key_constants.KEY_INSTANCES: 'test_instance',
48 | key_constants.KEY_LOGTYPES: 'test_log_type',
49 | key_constants.KEY_PARSER_EXTENSIONS: 'test_parserextension_id'
50 | }
51 |
--------------------------------------------------------------------------------
/parsers/parsers.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Grouping Parser CLI commands."""
16 |
17 | import click
18 |
19 | from parsers.commands import activate_parser
20 | from parsers.commands import archive
21 | from parsers.commands import classify_log_type
22 | from parsers.commands import deactivate_parser
23 | from parsers.commands import delete_extension
24 | from parsers.commands import delete_parser
25 | from parsers.commands import download
26 | from parsers.commands import generate
27 | from parsers.commands import get_extension
28 | from parsers.commands import get_parser
29 | from parsers.commands import get_validation_report
30 | from parsers.commands import history
31 | from parsers.commands import list # pylint: disable=redefined-builtin
32 | from parsers.commands import list_errors
33 | from parsers.commands import list_extensions
34 | from parsers.commands import list_parsers
35 | from parsers.commands import run
36 | from parsers.commands import run_parser
37 | from parsers.commands import status
38 | from parsers.commands import submit
39 | from parsers.commands import submit_extension
40 | from parsers.commands import submit_parser
41 |
42 |
43 | @click.group(name="parsers", help="Manage config based parsers")
44 | def parsers() -> None:
45 | """Group of commands to interact with Parser APIs."""
46 |
47 |
48 | parsers.add_command(activate_parser.activate_parser)
49 | parsers.add_command(archive.archive)
50 | parsers.add_command(classify_log_type.classify_log_type)
51 | parsers.add_command(deactivate_parser.deactivate_parser)
52 | parsers.add_command(delete_extension.delete_extension)
53 | parsers.add_command(delete_parser.delete_parser)
54 | parsers.add_command(download.download)
55 | parsers.add_command(generate.generate)
56 | parsers.add_command(get_extension.get_extension)
57 | parsers.add_command(get_parser.get_parser)
58 | parsers.add_command(get_validation_report.get_validation_report)
59 | parsers.add_command(history.history)
60 | parsers.add_command(list.list_command)
61 | parsers.add_command(list_errors.list_errors)
62 | parsers.add_command(list_parsers.list_parsers)
63 | parsers.add_command(list_extensions.list_extensions)
64 | parsers.add_command(run.run)
65 | parsers.add_command(run_parser.run_parser)
66 | parsers.add_command(status.status_command)
67 | parsers.add_command(submit.submit)
68 | parsers.add_command(submit_extension.submit_extension)
69 | parsers.add_command(submit_parser.submit_parser)
70 |
--------------------------------------------------------------------------------
/parsers/parsers_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Unit tests for parsers.py."""
16 |
17 | from click.testing import CliRunner
18 |
19 | from parsers.parsers import parsers
20 |
21 |
22 | runner = CliRunner()
23 |
24 |
25 | def test_parser() -> None:
26 | """Test case for parsers."""
27 | result = runner.invoke(parsers)
28 | expected_output = """Commands:
29 | activate_parser [New]Activate a parser
30 | archive Archives a parser given the config ID
31 | classify_log_type [New]Classify the provided logs to the log types.
32 | deactivate_parser [New]Deactivate a parser
33 | delete_extension [New]Delete an extension
34 | delete_parser [New]Delete a parser
35 | download Download parser code given config ID or log type
36 | generate Generate sample logs for a given log type
37 | get_extension [New]Get details of an extension
38 | get_parser [New]Get details of a parser
39 | get_validation_report [New]Get validation report for a parser/extension
40 | history History retrieves all parser submissions given a...
41 | list List all parsers of a given customer
42 | list_errors List errors of a log type between specific timestamps
43 | list_extensions [New]List all extensions for a given customer
44 | list_parsers [New]List all parsers for a given customer
45 | run Run the parser against given logs
46 | run_parser [New]Run a parser(with extension) against given logs
47 | status Get status of a submitted parser
48 | submit Submit a new parser
49 | submit_extension [New]Submit a new extension
50 | submit_parser [New]Submit a new parser"""
51 | assert expected_output in result.output
52 |
--------------------------------------------------------------------------------
/parsers/tests/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 |
--------------------------------------------------------------------------------
/parsers/url.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Return URLs to interact with CBN APIs."""
16 |
17 | from typing import Dict
18 | import urllib.parse
19 |
20 | from common import uri
21 |
22 | API_VERSION = 'v1'
23 | DATAPLANE_API_VERSION = 'v1alpha'
24 | HTTP_REQUEST_TIMEOUT_IN_SECS = 1200
25 | HTTP_REQUEST_HEADERS = {'Content-type': 'application/x-www-form-urlencoded'}
26 | PARENT = 'projects/{project}/locations/{location}/instances/{instance}/logTypes/{log_type}'
27 | PATH_DICT = {
28 | # Backstory APIs
29 | 'list': 'tools/cbnParsers',
30 | 'run': 'tools:validateCbnParser',
31 | 'history': 'tools/cbnParsers:listCbnParserHistory',
32 | 'generate': 'tools:retrieveSampleLogs',
33 | 'status': 'tools/cbnParsers',
34 | 'list_errors': 'tools/cbnParsers:listCbnParserErrors',
35 | # Dataplane APIs
36 | 'activate_parser': f'{PARENT}/parsers/{{parser}}:activate',
37 | 'deactivate_parser': f'{PARENT}/parsers/{{parser}}:deactivate',
38 | 'classify_log_type': 'projects/{project}/locations/{location}/instances/{instance}/logs:classify',
39 | 'delete_parser': f'{PARENT}/parsers/{{parser}}',
40 | 'delete_extension': f'{PARENT}/parserExtensions/{{parser_extension}}',
41 | 'get_parser': f'{PARENT}/parsers/{{parser}}',
42 | 'get_parser_validation_report': (
43 | f'{PARENT}/parsers/{{parser}}/validationReports/{{validationReport}}'),
44 | 'get_parserextension_validation_report': f'{PARENT}/parserExtensions/{{parserExtension}}/validationReports/{{validationReport}}',
45 | 'get_extension': f'{PARENT}/parserExtensions/{{parser_extension}}',
46 | 'list_parsers': f'{PARENT}/parsers',
47 | 'list_parser_parsing_errors': f'{PARENT}/parsers/{{parser}}/validationReports/{{validationReport}}/parsingErrors',
48 | 'list_parserextension_parsing_errors': f'{PARENT}/parserExtensions/{{parserExtension}}/validationReports/{{validationReport}}/parsingErrors',
49 | 'list_extensions': f'{PARENT}/parserExtensions',
50 | 'run_parser': f'{PARENT}:runParser',
51 | 'submit_parser': f'{PARENT}/parsers',
52 | 'submit_extension': f'{PARENT}/parserExtensions',
53 | }
54 |
55 |
56 | def get_url(region: str, command: str, environment: str,
57 | **query_params: Dict[str, str]) -> str:
58 | """Get URL for the given command.
59 |
60 | Args:
61 | region (str): Region (US, EUROPE, ASIA_SOUTHEAST1)
62 | command (str): Command name
63 | environment (str): Environment (prod, test)
64 | **query_params(Dict): Query
65 | parameters
66 |
67 | Returns:
68 | str: URL to interact with CBN APIs.
69 | """
70 | url = f'{uri.get_base_url(region, "", environment)}/{API_VERSION}/{PATH_DICT[command]}'
71 | if query_params:
72 | url += f'?{urllib.parse.urlencode(query_params)}'
73 | return url
74 |
75 |
76 | def get_dataplane_url(
77 | region: str,
78 | command: str,
79 | environment: str,
80 | resources: Dict[str, str],
81 | **query_params) -> str:
82 | """Get Dataplane URL for the given command.
83 |
84 | Args:
85 | region (str): Region (US, EUROPE, ASIA_SOUTHEAST1, EUROPE_WEST2)
86 | command (str): Command name
87 | environment (str): Environment (prod, test)
88 | resources (Dict): The resources for the URL
89 | **query_params: Optional keyword options for query parameters
90 |
91 | Returns:
92 | str: URL to interact with Chronicle APIs.
93 | """
94 | if region == 'EUROPE':
95 | region = 'eu'
96 | url = (
97 | f'{uri.get_dataplane_base_url(region.lower(), "", environment)}'
98 | f'/{DATAPLANE_API_VERSION}/{PATH_DICT[command].format(**resources)}')
99 | if query_params:
100 | url += f'?{urllib.parse.urlencode(query_params)}'
101 | return url
102 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | # coding=utf-8
2 | # Copyright 2022 Google LLC.
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | click==8.1.3
17 | colorama==0.4.0
18 | google-api-python-client==2.44.0
19 | pytest==7.2.0
20 | pytest-cov==3.0.0
21 | pyyaml==6.0.0
22 | stringcase==1.2.0
23 | wheel
24 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Metadata and requirements for building app package."""
16 | from setuptools import setup
17 | DEFAULT_REQUIREMENT_DOC = 'requirements.txt'
18 |
19 | with open(DEFAULT_REQUIREMENT_DOC) as f:
20 | deps = f.read().splitlines()
21 |
22 | setup(
23 | name='chronicle_cli',
24 | version='1.1',
25 | py_modules=['main'],
26 | install_requires=deps,
27 | entry_points="""
28 | [console_scripts]
29 | chronicle_cli=main:cli
30 | """,
31 | )
32 |
--------------------------------------------------------------------------------
/tools/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 |
--------------------------------------------------------------------------------
/tools/bigquery.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Grouping BigQuery CLI commands."""
16 |
17 | import click
18 |
19 | from tools.commands import provide_access
20 |
21 |
22 | @click.group(name="bigquery", help="Manage Big Query export")
23 | def bigquery() -> None:
24 | """Group of commands to interact with bigquery APIs."""
25 |
26 |
27 | bigquery.add_command(provide_access.provide_access)
28 |
--------------------------------------------------------------------------------
/tools/bigquery_templates.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Templates for printing output to console."""
16 |
17 | import string
18 |
19 | errors_response_template = string.Template("""\
20 |
21 | Error while providing access:
22 | Response code: ${error_code}
23 | Error: ${error_msg}""")
24 |
25 | success_response_template = string.Template("""\
26 |
27 | Access provided to email: ${email}""")
28 |
--------------------------------------------------------------------------------
/tools/bigquery_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Unit tests for bigquery.py."""
16 | from click.testing import CliRunner
17 | from tools.bigquery import bigquery
18 |
19 | runner = CliRunner()
20 |
21 |
22 | def test_bigquery() -> None:
23 | """Test case for bigquery."""
24 | result = runner.invoke(bigquery)
25 | expected_output = """Commands:
26 | provide_access Give permission to an email to access Big Query."""
27 | assert expected_output in result.output
28 |
--------------------------------------------------------------------------------
/tools/commands/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 |
--------------------------------------------------------------------------------
/tools/commands/provide_access.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Give permission to an email to view Big Query tables and run queries."""
16 |
17 | from typing import AnyStr
18 |
19 | import click
20 |
21 | from common import api_utility
22 | from common import chronicle_auth
23 | from common import exception_handler
24 | from common import options
25 | from common.constants import key_constants as common_constants
26 | from common.constants import status
27 | from tools import bigquery_templates
28 | from tools import url
29 | from tools.constants import key_constants as bigquery_constants
30 |
31 |
32 | @click.command(
33 | name="provide_access",
34 | help="Give permission to an email to access Big Query.")
35 | @options.env_option
36 | @options.region_option
37 | @options.verbose_option
38 | @options.credential_file_option
39 | @exception_handler.catch_exception()
40 | def provide_access(credential_file: AnyStr, verbose: bool, region: str,
41 | env: str) -> None:
42 | """Give permission to an email to access Big Query.
43 |
44 | Args:
45 | credential_file (AnyStr): Path of Service Account JSON.
46 | verbose (bool): Option for printing verbose output to console.
47 | region (str): Option for selecting regions. Available options - US, EUROPE,
48 | ASIA_SOUTHEAST1.
49 | env (str): Option for selection environment. Available options - prod, test.
50 |
51 | Raises:
52 | OSError: Failed to read the given file, e.g. not found, no read access
53 | (https://docs.python.org/library/exceptions.html#os-exceptions).
54 | ValueError: Invalid file contents.
55 | KeyError: Required key is not present in dictionary.
56 | TypeError: If response data is not JSON.
57 | """
58 | email = click.prompt("Enter email", show_default=False, default="")
59 |
60 | if not email:
61 | click.echo("Email not provided. Please enter email.")
62 | return
63 |
64 | click.echo("Providing Bigquery access...")
65 |
66 | provide_access_url = url.get_url(region, "provide_bq_access", env)
67 | client = chronicle_auth.initialize_http_session(credential_file)
68 | method = "PATCH"
69 | data = {bigquery_constants.KEY_EMAIL_ID: email}
70 | response = client.request(
71 | method,
72 | provide_access_url,
73 | timeout=url.HTTP_REQUEST_TIMEOUT_IN_SECS,
74 | data=data)
75 |
76 | parsed_response = api_utility.check_content_type(response.text)
77 |
78 | if response.status_code != status.STATUS_OK:
79 | error_response = bigquery_templates.errors_response_template.substitute(
80 | error_code=f"{response.status_code}",
81 | error_msg=f"{parsed_response[common_constants.KEY_ERROR][common_constants.KEY_MESSAGE]}"
82 | )
83 | click.echo(error_response)
84 | return
85 |
86 | success_response = bigquery_templates.success_response_template.substitute(
87 | email=f"{parsed_response[bigquery_constants.KEY_EMAIL_ID]}")
88 | click.echo(success_response)
89 |
90 | if verbose:
91 | api_utility.print_request_details(provide_access_url, method, None,
92 | parsed_response)
93 |
--------------------------------------------------------------------------------
/tools/commands/provide_access_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Tests for provide_access.py."""
16 |
17 | from unittest import mock
18 |
19 | from click.testing import CliRunner
20 |
21 | from common import uri
22 | from mock_test_utility import MockResponse
23 | from tools import url
24 | from tools.commands.provide_access import provide_access
25 | from tools.constants import key_constants as bigquery_constants
26 | from tools.tests.fixtures import * # pylint: disable=wildcard-import
27 |
28 | runner = CliRunner()
29 | TEST_PROVIDE_ACCESS_URL = f"{uri.BASE_URL}/tools/bigqueryAccess:update"
30 |
31 |
32 | @mock.patch(
33 | "common.chronicle_auth.initialize_http_session"
34 | )
35 | @mock.patch("tools.url.get_url")
36 | @mock.patch(
37 | "tools.commands.provide_access.click.prompt"
38 | )
39 | def test_provide_access_command(mock_input: mock.MagicMock,
40 | mock_url: mock.MagicMock,
41 | mock_client: mock.MagicMock,
42 | test_provide_access_data: MockResponse) -> None:
43 | """Test case to check response for status of command.
44 |
45 | Args:
46 | mock_input (mock.MagicMock): Mock prompt object
47 | mock_url (mock.MagicMock): Mock object
48 | mock_client (mock.MagicMock): Mock object
49 | test_provide_access_data (Tuple): Test input data
50 | """
51 | mock_url.return_value = TEST_PROVIDE_ACCESS_URL
52 | mock_input.side_effect = ["test_email_id@testcompany.com"]
53 | mock_client.return_value = mock.Mock()
54 | mock_client.return_value.request.side_effect = [test_provide_access_data]
55 | data = {bigquery_constants.KEY_EMAIL_ID: "test_email_id@testcompany.com"}
56 | result = runner.invoke(provide_access)
57 | assert """Providing Bigquery access...
58 |
59 | Access provided to email: test_email_id@testcompany.com
60 | """ in result.output
61 | mock_url.assert_called_once_with("US", "provide_bq_access", "prod")
62 | mock_client.return_value.request.assert_called_once_with(
63 | "PATCH",
64 | TEST_PROVIDE_ACCESS_URL,
65 | timeout=url.HTTP_REQUEST_TIMEOUT_IN_SECS,
66 | data=data)
67 | mock_input.assert_called_once_with(
68 | "Enter email", show_default=False, default="")
69 |
70 |
71 | @mock.patch(
72 | "common.chronicle_auth.initialize_http_session"
73 | )
74 | @mock.patch("tools.url.get_url")
75 | @mock.patch(
76 | "tools.commands.provide_access.click.prompt"
77 | )
78 | def test_provide_access_bigquery_500(mock_input: mock.MagicMock,
79 | mock_url: mock.MagicMock,
80 | mock_client: mock.MagicMock,
81 | test_500_resp: MockResponse) -> None:
82 | """Test case to check response for provide_access bigquery 500 response code.
83 |
84 | Args:
85 | mock_input (mock.MagicMock): Mock prompt object
86 | mock_url (mock.MagicMock): Mock object
87 | mock_client (mock.MagicMock): Mock object
88 | test_500_resp (Tuple): Test response data
89 | """
90 | mock_input.side_effect = ["test_email_id@testcompany.com"]
91 | mock_url.return_value = TEST_PROVIDE_ACCESS_URL
92 | mock_client.return_value = mock.Mock()
93 | mock_client.return_value.request.side_effect = [test_500_resp]
94 | result = runner.invoke(provide_access)
95 | assert """Providing Bigquery access...
96 |
97 | Error while providing access:
98 | Response code: 500
99 | Error: test error
100 | """ in result.output
101 |
102 |
103 | @mock.patch(
104 | "tools.commands.provide_access.click.prompt"
105 | )
106 | def test_provide_access_empty_email(mock_input: mock.MagicMock) -> None:
107 | """Test case to check the console output if no input provided for log type.
108 |
109 | Args:
110 | mock_input (mock.MagicMock): Mock object
111 | """
112 | mock_input.return_value = ""
113 | result = runner.invoke(provide_access)
114 | assert """Email not provided. Please enter email.""" in result.output
115 |
116 |
117 | def test_prompt_text() -> None:
118 | """Test case to check prompt text."""
119 | result = runner.invoke(provide_access)
120 | assert "Enter email:" in result.output
121 |
--------------------------------------------------------------------------------
/tools/constants/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 |
--------------------------------------------------------------------------------
/tools/constants/key_constants.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Common constants to be used for big query api."""
16 | KEY_EMAIL_ID = 'email'
17 |
--------------------------------------------------------------------------------
/tools/tests/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 |
--------------------------------------------------------------------------------
/tools/tests/fixtures.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Fixtures for bigquery commands."""
16 |
17 | import os
18 | from typing import Any
19 |
20 | import pytest
21 |
22 | from mock_test_utility import MockResponse
23 |
24 | TEST_DATA_DIR = os.path.dirname(__file__)
25 | TEMP_TEST_TXT_FILE = os.path.join(TEST_DATA_DIR, "test.txt")
26 | TEMP_TEST_JSON_FILE = os.path.join(TEST_DATA_DIR, "test.json")
27 | TEMP_CONF_FILE = os.path.join(TEST_DATA_DIR,
28 | "test_log_type_20220824062200.conf")
29 | TEMP_SUBMIT_CONF_FILE = os.path.join(TEST_DATA_DIR, "test_config_file.conf")
30 |
31 | os.system(f"chmod -R +rw {TEST_DATA_DIR}")
32 |
33 |
34 | def create_temp_config_file(file_path: str, content: str) -> None:
35 | """Creates temporary config file with the content.
36 |
37 | Args:
38 | file_path (str): Path to create the temp config file.
39 | content (str): content to be written in the file.
40 | """
41 | with open(file_path, "w") as file:
42 | if content:
43 | file.write(content)
44 |
45 |
46 | @pytest.fixture()
47 | def test_500_resp() -> MockResponse:
48 | """Test input data."""
49 | return MockResponse(
50 | status_code=500,
51 | text="""{"error": {"code": 500, "message": "test error"}}""")
52 |
53 |
54 | @pytest.fixture()
55 | def test_provide_access_data() -> MockResponse:
56 | """Test response data."""
57 | return MockResponse(
58 | status_code=200, text="""{"email": "test_email_id@testcompany.com"}""")
59 |
60 |
61 | @pytest.fixture(scope="function", autouse=True)
62 | def cleanup(request: Any):
63 | """Cleanup testing files once we are finished."""
64 |
65 | def remove_test_files():
66 | files = [
67 | TEMP_TEST_TXT_FILE, TEMP_TEST_JSON_FILE, TEMP_CONF_FILE,
68 | TEMP_SUBMIT_CONF_FILE
69 | ]
70 | for file_path in files:
71 | try:
72 | os.remove(file_path)
73 | except FileNotFoundError:
74 | pass
75 |
76 | request.addfinalizer(remove_test_files)
77 |
--------------------------------------------------------------------------------
/tools/url.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Return URLs to interact with BigQuery APIs."""
16 |
17 | from typing import Dict
18 | import urllib.parse
19 |
20 | from common import uri
21 |
22 | API_VERSION = 'v1'
23 | HTTP_REQUEST_TIMEOUT_IN_SECS = 1200
24 | HTTP_REQUEST_HEADERS = {'Content-type': 'application/x-www-form-urlencoded'}
25 | PATH_DICT = {
26 | 'provide_bq_access': 'tools/bigqueryAccess:update',
27 | }
28 |
29 |
30 | def get_url(region: str, command: str, environment: str,
31 | **query_params: Dict[str, str]) -> str:
32 | """Gets URL for the given command.
33 |
34 | Args:
35 | region (str): Region (US, EUROPE, ASIA_SOUTHEAST1)
36 | command (str): Command name
37 | environment (str): Environment (prod, test)
38 | **query_params(Dict): Query
39 | parameters
40 |
41 | Returns:
42 | str: URL to interact with CBN APIs.
43 | """
44 | url = f'{uri.get_base_url(region, "", environment)}/{API_VERSION}/{PATH_DICT[command]}'
45 | if query_params:
46 | url += f'?{urllib.parse.urlencode(query_params)}'
47 | return url
48 |
--------------------------------------------------------------------------------
/tools/url_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2022 Google LLC
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Unit tests for url.py."""
16 |
17 | from tools import url
18 |
19 |
20 | def test_get_url() -> None:
21 | """Test whether URL for big query commands are generated as expected."""
22 | # provide bigquery access commands.
23 | assert url.get_url(
24 | region="us", command="provide_bq_access", environment="prod"
25 | ) == "https://backstory.googleapis.com/v1/tools/bigqueryAccess:update"
26 | assert url.get_url(
27 | region="europe", command="provide_bq_access", environment="prod"
28 | ) == "https://europe-backstory.googleapis.com/v1/tools/bigqueryAccess:update"
29 | assert url.get_url(
30 | region="asia-southeast1", command="provide_bq_access", environment="prod"
31 | ) == "https://asia-southeast1-backstory.googleapis.com/v1/tools/bigqueryAccess:update"
32 | assert url.get_url(
33 | region="us", command="provide_bq_access", environment="test"
34 | ) == "https://test-backstory.sandbox.googleapis.com/v1/tools/bigqueryAccess:update"
35 | assert url.get_url(
36 | region="us",
37 | command="provide_bq_access",
38 | environment="prod",
39 | input="sample_test"
40 | ) == "https://backstory.googleapis.com/v1/tools/bigqueryAccess:update?input=sample_test"
41 |
--------------------------------------------------------------------------------