├── .github ├── dependabot.yml └── workflows │ ├── dependency_review.yml │ ├── format_check.yml │ ├── linting.yml │ └── unit_testing.yml ├── .gitignore ├── .pylintrc ├── LICENSE ├── README.md ├── config.ini ├── documentation ├── .DS_Store ├── architecture.md ├── contributing.md ├── images │ ├── example_architecture_macie.drawio.png │ └── logo.png └── talks │ ├── aws_user_group_chicago_2023 │ ├── README.md │ ├── aws_user_group_presentation_final.pptx │ └── feb_23_2023.webp │ ├── bsides_dfw_2022 │ ├── README.md │ ├── architecture_diagrams │ │ ├── dc_fss_macie_architecture_diagram_solution.drawio.png │ │ ├── mock_org_arc_diagram_complete.drawio.png │ │ ├── mock_org_arc_diagram_fss_expanded.drawio.png │ │ ├── mock_org_architecture_diagram.drawio.png │ │ ├── mock_org_architecture_diagram_with_dc.drawio.png │ │ ├── mock_org_architecture_diagram_with_dc_complete.drawio.png │ │ ├── mock_org_architecture_diagram_with_fss.drawio.png │ │ ├── mock_org_cloudone_architecture_diagram.drawio.png │ │ └── mock_org_macie_architecture_diagram.drawio.png │ └── bsides_dfw_2022_preso_final.pptx │ └── devopsdays_dallas_2022 │ ├── README.md │ ├── datacop_devopsdays_dallas_final.pptx │ └── images │ ├── architecture_diagram.drawio.png │ ├── architecture_diagram.png │ ├── architecture_diagram_revamp.drawio.png │ ├── architecture_diagram_solution.drawio.png │ ├── create_activate_venv.gif │ ├── datacop-benefits.drawio.png │ ├── example_arch_diagram_datacop.drawio.png │ ├── example_architecture_diagram.drawio.png │ ├── example_architecture_diagram_macie.drawio.png │ ├── install_deps.gif │ ├── manual_process.drawio.png │ └── manual_process.png ├── requirements.txt ├── sample-event.json ├── src ├── cdk-cloudformation │ ├── __init__.py │ ├── app.py │ ├── cdk.json │ ├── source.bat │ └── templates │ │ ├── __init__.py │ │ ├── constructs │ │ ├── __init__.py │ │ └── lambda_packager.py │ │ ├── core_stack.py │ │ └── integrations │ │ ├── __init__.py │ │ ├── aws_macie_stack.py │ │ └── file_storage_stack.py ├── lambda_func │ ├── __init__.py │ ├── data_cop │ │ ├── __init__.py │ │ ├── enums_.py │ │ ├── exceptions_.py │ │ ├── logging_config.py │ │ ├── parser_.py │ │ ├── s3_service.py │ │ ├── session_config.py │ │ ├── sfn_service.py │ │ ├── sns_service.py │ │ └── ssm_service.py │ ├── handler.py │ └── lambda_requirements.txt └── macie_setup.py ├── tasks.py └── tests ├── __init__.py ├── conftest.py ├── mocks └── 251d94c9-34f8-3ac0-b3e3-35714dc60760.jsonl.gz └── test_s3_service.py /.github/dependabot.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/.github/dependabot.yml -------------------------------------------------------------------------------- /.github/workflows/dependency_review.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/.github/workflows/dependency_review.yml -------------------------------------------------------------------------------- /.github/workflows/format_check.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/.github/workflows/format_check.yml -------------------------------------------------------------------------------- /.github/workflows/linting.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/.github/workflows/linting.yml -------------------------------------------------------------------------------- /.github/workflows/unit_testing.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/.github/workflows/unit_testing.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/.gitignore -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/.pylintrc -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/README.md -------------------------------------------------------------------------------- /config.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/config.ini -------------------------------------------------------------------------------- /documentation/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/.DS_Store -------------------------------------------------------------------------------- /documentation/architecture.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/architecture.md -------------------------------------------------------------------------------- /documentation/contributing.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/contributing.md -------------------------------------------------------------------------------- /documentation/images/example_architecture_macie.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/images/example_architecture_macie.drawio.png -------------------------------------------------------------------------------- /documentation/images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/images/logo.png -------------------------------------------------------------------------------- /documentation/talks/aws_user_group_chicago_2023/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/aws_user_group_chicago_2023/README.md -------------------------------------------------------------------------------- /documentation/talks/aws_user_group_chicago_2023/aws_user_group_presentation_final.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/aws_user_group_chicago_2023/aws_user_group_presentation_final.pptx -------------------------------------------------------------------------------- /documentation/talks/aws_user_group_chicago_2023/feb_23_2023.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/aws_user_group_chicago_2023/feb_23_2023.webp -------------------------------------------------------------------------------- /documentation/talks/bsides_dfw_2022/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/bsides_dfw_2022/README.md -------------------------------------------------------------------------------- /documentation/talks/bsides_dfw_2022/architecture_diagrams/dc_fss_macie_architecture_diagram_solution.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/bsides_dfw_2022/architecture_diagrams/dc_fss_macie_architecture_diagram_solution.drawio.png -------------------------------------------------------------------------------- /documentation/talks/bsides_dfw_2022/architecture_diagrams/mock_org_arc_diagram_complete.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/bsides_dfw_2022/architecture_diagrams/mock_org_arc_diagram_complete.drawio.png -------------------------------------------------------------------------------- /documentation/talks/bsides_dfw_2022/architecture_diagrams/mock_org_arc_diagram_fss_expanded.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/bsides_dfw_2022/architecture_diagrams/mock_org_arc_diagram_fss_expanded.drawio.png -------------------------------------------------------------------------------- /documentation/talks/bsides_dfw_2022/architecture_diagrams/mock_org_architecture_diagram.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/bsides_dfw_2022/architecture_diagrams/mock_org_architecture_diagram.drawio.png -------------------------------------------------------------------------------- /documentation/talks/bsides_dfw_2022/architecture_diagrams/mock_org_architecture_diagram_with_dc.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/bsides_dfw_2022/architecture_diagrams/mock_org_architecture_diagram_with_dc.drawio.png -------------------------------------------------------------------------------- /documentation/talks/bsides_dfw_2022/architecture_diagrams/mock_org_architecture_diagram_with_dc_complete.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/bsides_dfw_2022/architecture_diagrams/mock_org_architecture_diagram_with_dc_complete.drawio.png -------------------------------------------------------------------------------- /documentation/talks/bsides_dfw_2022/architecture_diagrams/mock_org_architecture_diagram_with_fss.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/bsides_dfw_2022/architecture_diagrams/mock_org_architecture_diagram_with_fss.drawio.png -------------------------------------------------------------------------------- /documentation/talks/bsides_dfw_2022/architecture_diagrams/mock_org_cloudone_architecture_diagram.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/bsides_dfw_2022/architecture_diagrams/mock_org_cloudone_architecture_diagram.drawio.png -------------------------------------------------------------------------------- /documentation/talks/bsides_dfw_2022/architecture_diagrams/mock_org_macie_architecture_diagram.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/bsides_dfw_2022/architecture_diagrams/mock_org_macie_architecture_diagram.drawio.png -------------------------------------------------------------------------------- /documentation/talks/bsides_dfw_2022/bsides_dfw_2022_preso_final.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/bsides_dfw_2022/bsides_dfw_2022_preso_final.pptx -------------------------------------------------------------------------------- /documentation/talks/devopsdays_dallas_2022/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/devopsdays_dallas_2022/README.md -------------------------------------------------------------------------------- /documentation/talks/devopsdays_dallas_2022/datacop_devopsdays_dallas_final.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/devopsdays_dallas_2022/datacop_devopsdays_dallas_final.pptx -------------------------------------------------------------------------------- /documentation/talks/devopsdays_dallas_2022/images/architecture_diagram.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/devopsdays_dallas_2022/images/architecture_diagram.drawio.png -------------------------------------------------------------------------------- /documentation/talks/devopsdays_dallas_2022/images/architecture_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/devopsdays_dallas_2022/images/architecture_diagram.png -------------------------------------------------------------------------------- /documentation/talks/devopsdays_dallas_2022/images/architecture_diagram_revamp.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/devopsdays_dallas_2022/images/architecture_diagram_revamp.drawio.png -------------------------------------------------------------------------------- /documentation/talks/devopsdays_dallas_2022/images/architecture_diagram_solution.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/devopsdays_dallas_2022/images/architecture_diagram_solution.drawio.png -------------------------------------------------------------------------------- /documentation/talks/devopsdays_dallas_2022/images/create_activate_venv.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/devopsdays_dallas_2022/images/create_activate_venv.gif -------------------------------------------------------------------------------- /documentation/talks/devopsdays_dallas_2022/images/datacop-benefits.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/devopsdays_dallas_2022/images/datacop-benefits.drawio.png -------------------------------------------------------------------------------- /documentation/talks/devopsdays_dallas_2022/images/example_arch_diagram_datacop.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/devopsdays_dallas_2022/images/example_arch_diagram_datacop.drawio.png -------------------------------------------------------------------------------- /documentation/talks/devopsdays_dallas_2022/images/example_architecture_diagram.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/devopsdays_dallas_2022/images/example_architecture_diagram.drawio.png -------------------------------------------------------------------------------- /documentation/talks/devopsdays_dallas_2022/images/example_architecture_diagram_macie.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/devopsdays_dallas_2022/images/example_architecture_diagram_macie.drawio.png -------------------------------------------------------------------------------- /documentation/talks/devopsdays_dallas_2022/images/install_deps.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/devopsdays_dallas_2022/images/install_deps.gif -------------------------------------------------------------------------------- /documentation/talks/devopsdays_dallas_2022/images/manual_process.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/devopsdays_dallas_2022/images/manual_process.drawio.png -------------------------------------------------------------------------------- /documentation/talks/devopsdays_dallas_2022/images/manual_process.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/documentation/talks/devopsdays_dallas_2022/images/manual_process.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/requirements.txt -------------------------------------------------------------------------------- /sample-event.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/sample-event.json -------------------------------------------------------------------------------- /src/cdk-cloudformation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/cdk-cloudformation/app.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/src/cdk-cloudformation/app.py -------------------------------------------------------------------------------- /src/cdk-cloudformation/cdk.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/src/cdk-cloudformation/cdk.json -------------------------------------------------------------------------------- /src/cdk-cloudformation/source.bat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/src/cdk-cloudformation/source.bat -------------------------------------------------------------------------------- /src/cdk-cloudformation/templates/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/cdk-cloudformation/templates/constructs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/cdk-cloudformation/templates/constructs/lambda_packager.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/src/cdk-cloudformation/templates/constructs/lambda_packager.py -------------------------------------------------------------------------------- /src/cdk-cloudformation/templates/core_stack.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/src/cdk-cloudformation/templates/core_stack.py -------------------------------------------------------------------------------- /src/cdk-cloudformation/templates/integrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/cdk-cloudformation/templates/integrations/aws_macie_stack.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/src/cdk-cloudformation/templates/integrations/aws_macie_stack.py -------------------------------------------------------------------------------- /src/cdk-cloudformation/templates/integrations/file_storage_stack.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/src/cdk-cloudformation/templates/integrations/file_storage_stack.py -------------------------------------------------------------------------------- /src/lambda_func/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/lambda_func/data_cop/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/lambda_func/data_cop/enums_.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/src/lambda_func/data_cop/enums_.py -------------------------------------------------------------------------------- /src/lambda_func/data_cop/exceptions_.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/src/lambda_func/data_cop/exceptions_.py -------------------------------------------------------------------------------- /src/lambda_func/data_cop/logging_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/src/lambda_func/data_cop/logging_config.py -------------------------------------------------------------------------------- /src/lambda_func/data_cop/parser_.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/src/lambda_func/data_cop/parser_.py -------------------------------------------------------------------------------- /src/lambda_func/data_cop/s3_service.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/src/lambda_func/data_cop/s3_service.py -------------------------------------------------------------------------------- /src/lambda_func/data_cop/session_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/src/lambda_func/data_cop/session_config.py -------------------------------------------------------------------------------- /src/lambda_func/data_cop/sfn_service.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/src/lambda_func/data_cop/sfn_service.py -------------------------------------------------------------------------------- /src/lambda_func/data_cop/sns_service.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/src/lambda_func/data_cop/sns_service.py -------------------------------------------------------------------------------- /src/lambda_func/data_cop/ssm_service.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/src/lambda_func/data_cop/ssm_service.py -------------------------------------------------------------------------------- /src/lambda_func/handler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/src/lambda_func/handler.py -------------------------------------------------------------------------------- /src/lambda_func/lambda_requirements.txt: -------------------------------------------------------------------------------- 1 | boto3 2 | botocore -------------------------------------------------------------------------------- /src/macie_setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/src/macie_setup.py -------------------------------------------------------------------------------- /tasks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/tasks.py -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | sys.path.append("./src/lambda_func") 4 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/tests/conftest.py -------------------------------------------------------------------------------- /tests/mocks/251d94c9-34f8-3ac0-b3e3-35714dc60760.jsonl.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/tests/mocks/251d94c9-34f8-3ac0-b3e3-35714dc60760.jsonl.gz -------------------------------------------------------------------------------- /tests/test_s3_service.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/damienjburks/DataCop/HEAD/tests/test_s3_service.py --------------------------------------------------------------------------------