├── .appveyor.yml ├── .buildkite └── pipeline.yml ├── .checkov.yaml ├── .circleci └── config.yml ├── .cirrus.yml ├── .coafile ├── .drone.yml ├── .editorconfig ├── .envrc ├── .envrc-python ├── .flake8 ├── .gitallowed ├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE.md └── workflows │ ├── *_centos.yaml.disabled │ ├── alpine.yaml │ ├── alpine_3.yaml │ ├── centos.yaml.disabled │ ├── centos7.yaml.disabled │ ├── centos8.yaml.disabled │ ├── checkov.yaml │ ├── codeowners.yaml │ ├── codeql-analysis.yml │ ├── debian.yaml │ ├── debian_10.yaml │ ├── debian_11.yaml │ ├── debian_12.yaml │ ├── debian_6.yaml.disabled │ ├── debian_7.yaml.disabled │ ├── debian_8.yaml.disabled │ ├── debian_9.yaml.disabled │ ├── docker_pytools_alpine.yaml │ ├── docker_pytools_debian.yaml │ ├── docker_pytools_fedora.yaml │ ├── docker_pytools_ubuntu.yaml │ ├── fedora.yaml │ ├── fork-sync.yaml │ ├── fork-update-pr.yaml │ ├── ghcr_python_ubuntu.yaml.disabled │ ├── grype.yaml │ ├── json.yaml │ ├── kics.yaml │ ├── mac.yaml │ ├── mac_11.yaml │ ├── mac_12.yaml │ ├── markdown.yaml │ ├── pypy2.yaml.disabled │ ├── pypy3.yaml.disabled │ ├── python2.7.yaml.disabled │ ├── python3.10.yaml │ ├── python3.11.yaml │ ├── python3.6.yaml.disabled │ ├── python3.7.yaml │ ├── python3.8.yaml │ ├── python3.9.yaml │ ├── semgrep-cloud.yaml │ ├── semgrep.yaml │ ├── shellcheck.yaml │ ├── trivy.yaml │ ├── ubuntu.yaml │ ├── ubuntu_14.04.yaml.disabled │ ├── ubuntu_16.04.yaml.disabled │ ├── ubuntu_18.04.yaml.disabled │ ├── ubuntu_20.04.yaml │ ├── ubuntu_22.04.yaml │ ├── ubuntu_github.yaml │ ├── validate.yaml │ ├── xml.yaml │ └── yaml.yaml ├── .gitignore ├── .gitlab-ci.yml ├── .gitmodules ├── .ipython-notebook-pyspark.00-pyspark-setup.py ├── .ipython-notebook-pyspark.ipython_notebook_config.py.j2 ├── .mdl.rb ├── .mdlrc ├── .pre-commit-config.yaml ├── .pylintrc ├── .pyup.yml ├── .semaphore └── semaphore.yml ├── .sonarcloud.properties ├── .travis.yml ├── .yamllint ├── Jenkinsfile ├── LICENSE ├── Makefile ├── README.md ├── ambari_ams_metrics.sh ├── ambari_blueprints.py ├── ambari_blueprints ├── README.md ├── hdp-2.3-cluster-basic-named.json ├── hdp-2.3-cluster-basic.json ├── hdp-2.3-cluster-hdfs-ha.json ├── hdp-2.3-cluster-hdfs-yarn-ha.json ├── hdp-2.3-cluster-hdfs-yarn-hbase-ha.json ├── hostmappings-hdfs-ha-random-hosts.json ├── hostmappings-hdfs-ha.json ├── hostmappings-hdfs-yarn-ha-random-hosts.json ├── hostmappings-hdfs-yarn-ha.json ├── hostmappings-hdfs-yarn-hbase-ha-random-hosts.json ├── hostmappings-hdfs-yarn-hbase-ha.json ├── hostmappings-named-random-hosts.json ├── hostmappings-named.json ├── hostmappings-random-hosts.json └── hostmappings.json ├── ambari_cancel_all_requests.sh ├── ambari_trigger_service_checks.py ├── anonymize.py ├── anonymize_custom.conf ├── anonymize_ignore.conf ├── anonymize_parallel.sh ├── aws_s3_presign.py ├── aws_users_access_key_age.py ├── aws_users_last_used.py ├── aws_users_pw_last_used.py ├── aws_users_unused_access_keys.py ├── azure-pipelines.yml ├── bitbucket-pipelines.yml ├── boot ├── buddy.yml ├── center.py ├── cicd ├── .concourse.yml ├── .gocd.yml ├── buildspec.yml └── cloudbuild.yaml ├── cloudera_navigator_tables_used.py ├── cloudera_navigator_tables_used_postgres.py ├── cloudformation ├── README.md ├── centos7-12nodes-encrypted.json ├── centos7-12nodes.json ├── centos7-1node-encrypted.json ├── centos7-1node.json ├── centos7-3nodes-encrypted.json └── centos7-3nodes.json ├── codefresh.yml ├── crunch_accounting_csv_statement_converter.py ├── crunch_accounting_csv_statement_converter_scripts ├── barclaycard_crunch_accounting_csv_statement_converter.sh ├── barclays_crunch_accounting_csv_statement_converter.sh └── lib.sh ├── docker_pull_all_images.sh ├── docker_pull_all_images_all_tags.sh ├── docker_pull_all_tags.sh ├── docker_registry_show_tags.py ├── dockerfiles_check_git_branches.py ├── dockerfiles_check_git_tags.py ├── dockerhub_search.py ├── dockerhub_show_tags.py ├── find_active_apache_drill.py ├── find_active_apache_drill2.py ├── find_active_cassandra.py ├── find_active_elasticsearch.py ├── find_active_hadoop2_namenode.py ├── find_active_hadoop_namenode.py ├── find_active_hadoop_yarn_resource_manager.py ├── find_active_hbase_master.py ├── find_active_hbase_stargate.py ├── find_active_hbase_thrift.py ├── find_active_impala.py ├── find_active_impala_catalog.py ├── find_active_impala_statestore.py ├── find_active_kubernetes_api.py ├── find_active_oozie.py ├── find_active_presto_coordinator.py ├── find_active_server.py ├── find_active_solrcloud.py ├── find_duplicate_files.py ├── find_missing_files_in_sequence.py ├── gcp_cloud_function_ifconfig ├── .gcloudignore ├── Makefile ├── README.md ├── deploy.sh ├── main.py └── requirements.txt ├── gcp_cloud_function_proxy ├── .gcloudignore ├── Makefile ├── README.md ├── deploy.sh ├── main.py └── requirements.txt ├── gcp_cloud_function_sql_export ├── .gcloudignore ├── Makefile ├── README.md ├── deploy.sh ├── main.py ├── requirements.txt └── serverless.yml ├── gcp_service_account_credential_keys.py ├── getent.py ├── git_check_branches_upstream.py ├── hbase_compact_tables.py ├── hbase_flush_tables.py ├── hbase_generate_data.py ├── hbase_region_requests.py ├── hbase_regions_by_memstore_size.py ├── hbase_regions_by_size.py ├── hbase_regions_least_used.py ├── hbase_regionserver_requests.py ├── hbase_scan_table_column_names.sh ├── hbase_show_table_region_ranges.py ├── hbase_table_region_row_distribution.py ├── hbase_table_regions_by_regionserver.sh ├── hbase_table_regionserver_distribution.sh ├── hbase_table_row_key_distribution.py ├── hbase_uncompacted_regions.sh ├── hdfs_files_native_checksums.jy ├── hdfs_files_stats.jy ├── hdfs_find_replication_factor_1.py ├── hdfs_time_block_reads.jy ├── headtail.py ├── hexanonymize.py ├── hive_compute_table_stats.py ├── hive_foreach_table.py ├── hive_schemas_csv.py ├── hive_tables_column_counts.py ├── hive_tables_list.py ├── hive_tables_locations.py ├── hive_tables_metadata.py ├── hive_tables_null_columns.py ├── hive_tables_null_rows.py ├── hive_tables_row_column_counts.py ├── hive_tables_row_counts.py ├── hive_tables_row_counts_any_nulls.py ├── impala_compute_table_stats.py ├── impala_foreach_table.py ├── impala_schemas_csv.py ├── impala_tables_column_counts.py ├── impala_tables_list.py ├── impala_tables_locations.py ├── impala_tables_metadata.py ├── impala_tables_null_columns.py ├── impala_tables_null_rows.py ├── impala_tables_row_column_counts.py ├── impala_tables_row_counts.py ├── impala_tables_row_counts_any_nulls.py ├── ipython_notebook_pyspark.py ├── json_docs_to_bulk_multiline.py ├── json_to_xml.py ├── json_to_yaml.py ├── jython_autoinstall.exp ├── jython_install.sh ├── kics.config ├── lib ├── __init__.py ├── hive_impala_cli.py └── postgres_cli.py ├── opentsdb_import_metric_distribution.py ├── opentsdb_list_metrics.sh ├── opentsdb_list_metrics_hbase.sh ├── pig-text-to-elasticsearch.pig ├── pig-text-to-solr.pig ├── pig_udfs.jy ├── plot_uk_marriage_rates.py ├── python_find_library_path.py ├── pythonpath.py ├── quay_show_tags.py ├── requirements.txt ├── selenium_hub_browser_test.py ├── serf_event_handler.py ├── setup ├── apk-packages-dev.txt ├── apk-packages-pip.txt ├── apk-packages.txt ├── bootstrap.sh ├── brew-packages.txt ├── ci_bootstrap.sh ├── ci_git_set_dir_safe.sh ├── deb-packages-dev.txt ├── deb-packages-optional.txt ├── deb-packages-pip.txt ├── deb-packages.txt ├── gocd_config_repo.json ├── jenkins-job.xml ├── rpm-packages-dev.txt ├── rpm-packages-optional.txt ├── rpm-packages-pip.txt └── rpm-packages.txt ├── sonar-project.properties ├── spark_avro_to_parquet.py ├── spark_csv_to_avro.py ├── spark_csv_to_parquet.py ├── spark_json_to_avro.py ├── spark_json_to_parquet.py ├── spark_parquet_to_avro.py ├── strip_ansi_escape_codes.py ├── teamcity ├── .teamcity.vcs.json ├── .teamcity.vcs.oauth.json └── .teamcity.vcs.ssh.json ├── tests ├── all.sh ├── check.sh ├── compile.sh ├── data │ ├── add_ou.ldif │ ├── cd_catalog.xml │ ├── embedded_double_quotes.json │ ├── header.csv │ ├── header_quoted.csv │ ├── ldap.ldif │ ├── ldap_download.sh │ ├── ldap_upload.sh │ ├── multirecord.json │ ├── multirecord_single_double_mixed_quotes.notjson │ ├── multirecord_single_quotes.notjson │ ├── multirecord_single_quotes_embedded_double_quotes.notjson │ ├── multirecord_single_quotes_embedded_double_quotes_unescaped.notjson │ ├── multirecord_with_blank_lines.notjson │ ├── plant_catalog.xml │ ├── quoted.csv │ ├── simple.xml │ ├── single_quotes.notjson │ ├── single_quotes_embedded_double_quotes.notjson │ ├── single_quotes_embedded_double_quotes_unescaped.notjson │ ├── test.csv │ ├── test.ini │ ├── test.ini-colons │ ├── test.ini-hashes │ ├── test.json │ ├── test.toml │ ├── test.yaml │ └── uk_marriage_rates_2022.xslx ├── docker │ ├── apache-drill-docker-compose.yml │ ├── common.yml │ ├── elasticsearch-docker-compose.yml │ ├── elasticsearch-elastic.co-docker-compose.yml │ ├── hadoop-docker-compose.yml │ ├── hbase-docker-compose.yml │ ├── presto-dev-docker-compose.yml │ ├── presto-docker-compose.yml │ ├── registry-docker-compose.yml │ └── solrcloud-docker-compose.yml ├── excluded.sh ├── help.sh ├── python3.sh ├── syntax.sh ├── test_anonymize.py ├── test_anonymize.sh ├── test_apache-drill.sh ├── test_center.sh ├── test_docker.sh ├── test_docker_registry_show_tags.sh ├── test_dockerfiles_check_git_branches.sh ├── test_dockerfiles_check_git_tags.sh ├── test_dockerhub_search.sh ├── test_dockerhub_show_tags.sh ├── test_elasticsearch.sh ├── test_find_active_server.sh ├── test_find_duplicate_files.sh ├── test_find_python_library_path.sh ├── test_getent.sh ├── test_git_check_branches_upstream.sh ├── test_hadoop.sh ├── test_hbase.sh ├── test_headtail.sh ├── test_hexanonymize.sh ├── test_json.sh ├── test_json_docs_to_bulk_multiline.sh ├── test_json_to_xml.sh ├── test_json_to_yaml.sh ├── test_opentsdb.sh ├── test_presto.sh ├── test_quay_show_tags.sh ├── test_serf_event_handler.sh ├── test_solrcloud.sh ├── test_spark_csv_to_avro.sh ├── test_spark_csv_to_parquet.sh ├── test_spark_json_to_avro.sh ├── test_spark_json_to_parquet.sh ├── test_spark_z_avro_to_parquet.sh ├── test_spark_z_parquet_to_avro.sh ├── test_strip_ansi_escape_codes.sh ├── test_timeout.sh ├── test_validate_avro.sh ├── test_validate_cson.sh ├── test_validate_csv.sh ├── test_validate_ini.sh ├── test_validate_json.sh ├── test_validate_ldap_ldif.sh ├── test_validate_multimedia.sh ├── test_validate_parquet.sh ├── test_validate_toml.sh ├── test_validate_xml.sh ├── test_validate_yaml.sh ├── test_welcome.sh ├── test_xml_to_json.sh ├── test_xml_to_yaml.sh ├── test_yamllint.sh └── utils.sh ├── timeout.py ├── travis_debug_session.py ├── travis_last_log.py ├── urldecode.py ├── urlencode.py ├── validate_avro.py ├── validate_cson.py ├── validate_csv.py ├── validate_ini.py ├── validate_ini2.py ├── validate_json.py ├── validate_ldap_ldif.py ├── validate_multimedia.py ├── validate_parquet.py ├── validate_properties.py ├── validate_toml.py ├── validate_xml.py ├── validate_yaml.py ├── welcome.py ├── xml_to_json.py ├── xml_to_yaml.py └── yaml_to_json.py /.buildkite/pipeline.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2020-03-13 21:10:39 +0000 (Fri, 13 Mar 2020) 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | # ============================================================================ # 17 | # B u i l d K i t e C I 18 | # ============================================================================ # 19 | 20 | # BuildKite Pipeline 21 | # 22 | # add this command to the UI and it will read the rest of the steps from here: 23 | # 24 | # - command: buildkite-agent pipeline upload 25 | 26 | # Yaml Anchors workaround to BuildKite's lack of global retries configuration - credit to Jason @ BuildKite for this workaround: 27 | # 28 | # https://forum.buildkite.community/t/reschedule-builds-on-other-agents-rather-than-fail-builds-when-agents-time-out-or-are-killed-machine-shut-down-or-put-to-sleep/1388/5 29 | # 30 | anchors: 31 | std_retries: &std_retries 32 | retry: 33 | automatic: 34 | - exit_status: -1 # Agent was lost 35 | limit: 2 36 | - exit_status: 255 # Forced agent shutdown 37 | limit: 2 38 | 39 | steps: 40 | - command: setup/ci_bootstrap.sh 41 | label: ci bootstrap 42 | timeout: 30 # brew can take 10 mins just to do a brew update 43 | branches: master 44 | <<: [*std_retries] 45 | - wait 46 | - command: make init 47 | label: init 48 | timeout: 2 49 | branches: master 50 | <<: [*std_retries] 51 | - wait 52 | - command: make ci 53 | label: build 54 | timeout: 60 55 | branches: master 56 | <<: [*std_retries] 57 | - wait 58 | - command: make test 59 | label: test 60 | timeout: 120 61 | branches: master 62 | <<: [*std_retries] 63 | -------------------------------------------------------------------------------- /.checkov.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2022-02-21 16:53:29 +0000 (Mon, 21 Feb 2022) 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | # ============================================================================ # 17 | # C h e c k o v c o n f i g 18 | # ============================================================================ # 19 | 20 | # https://github.com/bridgecrewio/checkov#configuration-using-a-config-file 21 | # 22 | # This is not well documented but the fields seem to be the same as: 23 | # 24 | # checkov --help 25 | # 26 | # See master template at: 27 | # 28 | # https://github.com/HariSekhon/Templates/blob/master/.checkov.yaml 29 | 30 | --- 31 | compact: true 32 | directory: 33 | - . 34 | skip-path: 35 | - bash-tools 36 | - pylib 37 | - sql 38 | - templates 39 | docker-image: harisekhon/pytools 40 | download-external-modules: true # without this gets lots of annoying warning lines such as '2022-02-22 16:14:40,180 [MainThread ] [WARNI] Failed to download module x/y/z:n.n.n' 41 | framework: 42 | - all 43 | no-guide: true 44 | output: cli 45 | quiet: true 46 | repo-id: HariSekhon/DevOps-Python-tools # what to report to Bridgecrew Cloud - without this gets annoying duplicate repos such as 'harisekhon_cli_repo/pytools' 47 | skip-suppressions: true 48 | soft-fail: true 49 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2020-02-23 23:30:14 +0000 (Sun, 23 Feb 2020) 4 | # Original: H1 2016 (Circle CI 1.x) 5 | # 6 | # vim:ts=2:sts=2:sw=2:et 7 | # 8 | # https://github.com/HariSekhon/DevOps-Python-tools 9 | # 10 | # License: see accompanying Hari Sekhon LICENSE file 11 | # 12 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 13 | # 14 | # https://www.linkedin.com/in/HariSekhon 15 | # 16 | 17 | # ============================================================================ # 18 | # C i r c l e C I 19 | # ============================================================================ # 20 | 21 | # Master Template with more advanced config: 22 | # 23 | # https://github.com/HariSekhon/Templates/blob/master/circleci_config.yml 24 | 25 | # Reference: 26 | # 27 | # https://circleci.com/docs/2.0/configuration-reference 28 | 29 | version: 2.1 30 | 31 | workflows: 32 | version: 2 33 | workflow: 34 | jobs: 35 | - build 36 | 37 | jobs: 38 | build: 39 | docker: 40 | - image: cimg/base:2024.12 41 | resource_class: small 42 | steps: 43 | # CLI is too old - config validate breaks in test - install new version to fix 44 | # doesn't work - existing version is too old to update 45 | #- run: circleci update 46 | - run: | 47 | curl -sSLf https://raw.githubusercontent.com/CircleCI-Public/circleci-cli/main/install.sh | sudo bash 48 | - checkout 49 | #- setup_remote_docker: 50 | # version: 20.10.11 51 | - run: setup/ci_bootstrap.sh 52 | - run: make init 53 | - run: make 54 | - run: make test 55 | -------------------------------------------------------------------------------- /.cirrus.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2020-02-24 16:55:36 +0000 (Mon, 24 Feb 2020) 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | # ============================================================================ # 17 | # C i r r u s C I 18 | # ============================================================================ # 19 | 20 | # https://cirrus-ci.org/guide/writing-tasks/ 21 | 22 | container: 23 | image: ubuntu:18.04 24 | 25 | task: 26 | env: 27 | TMPDIR: /var/tmp 28 | script: 29 | - setup/ci_bootstrap.sh 30 | - make init 31 | - make ci test 32 | -------------------------------------------------------------------------------- /.coafile: -------------------------------------------------------------------------------- 1 | [Default] 2 | bears = PEP8Bear,PyLintBear 3 | files = **/*.py 4 | -------------------------------------------------------------------------------- /.drone.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # XXX: putting this separator further down with code causes a parsing bug in drone lint 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2020-02-29 12:05:52 +0000 (Sat, 29 Feb 2020) 6 | # 7 | # vim:ts=2:sts=2:sw=2:et 8 | # 9 | # https://github.com/HariSekhon/DevOps-Python-tools 10 | # 11 | # License: see accompanying Hari Sekhon LICENSE file 12 | # 13 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 14 | # 15 | # https://www.linkedin.com/in/HariSekhon 16 | # 17 | 18 | # ============================================================================ # 19 | # D r o n e C I 20 | # ============================================================================ # 21 | 22 | # https://docs.drone.io/quickstart/cli/ 23 | # 24 | # https://docs.drone.io/cli/install/ 25 | # 26 | # brew install drone-cli 27 | # 28 | # cd to this directory 29 | # 30 | # drone exec [--pipeline default] [--include=thisstep] [--exclude=thatstep] 31 | 32 | kind: pipeline 33 | type: docker 34 | name: default 35 | 36 | steps: 37 | - name: build 38 | image: ubuntu:18.04 39 | #environment: 40 | # DEBUG: 1 41 | commands: 42 | - setup/ci_bootstrap.sh 43 | - make init 44 | - make ci 45 | - make test 46 | 47 | trigger: 48 | branch: 49 | - master 50 | -------------------------------------------------------------------------------- /.envrc-python: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: Mon Feb 22 17:42:01 2021 +0000 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | # ============================================================================ # 17 | # P y t h o n D i r E n v 18 | # ============================================================================ # 19 | 20 | # .envrc to auto-load the virtualenv inside the 'venv' directory if present 21 | 22 | # https://direnv.net/man/direnv-stdlib.1.html 23 | 24 | set -euo pipefail 25 | [ -n "${DEBUG:-}" ] && set -x 26 | #srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 27 | 28 | # this is necessary because newer versions of pip no longer allow you to install PyPI packages in system-packages by default 29 | for venv in "$PWD/venv" "$HOME/venv"; do 30 | if [ -f "$venv/bin/activate" ]; then 31 | echo 32 | echo "Virtualenv directory found in: $venv" 33 | echo 34 | echo "Activating Virtualenv inside the directory: $venv" 35 | 36 | # shellcheck disable=SC1091 37 | source "$venv/bin/activate" 38 | break 39 | fi 40 | done 41 | 42 | # read .env too 43 | #dotenv 44 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: Mon Oct 21 15:57:10 2019 +0100 4 | # 5 | # vim:ts=4:sts=4:sw=4:et 6 | # 7 | # https///github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | # ============================================================================ # 17 | # F l a k e 8 C o n f i g 18 | # ============================================================================ # 19 | 20 | # https://flake8.pycqa.org/en/latest/user/configuration.html 21 | 22 | [flake8] 23 | 24 | max-line-length = 120 25 | 26 | ignore = E265, 27 | E402, 28 | F401 29 | 30 | exclude = test*/* 31 | 32 | max-complexity = 10 33 | -------------------------------------------------------------------------------- /.gitallowed: -------------------------------------------------------------------------------- 1 | AKIAIOSFODNN7EXAMPLE 2 | 3 | wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY 4 | 5 | AQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT+FvwqnKwRcOIfrRh3c/LTo6UDdyJwOOvEVPvLXCrrrUtdnniCEXAMPLE/IvU1dYUg2RVAJBanLiHb4IgRmpRV3zrkuWJOgQs8IZZaIv2BXIa2R4OlgkBN9bkUDNCJiBeb/AXlzBBko7b15fjrBs2+cTQtpZ3CYWFXG8C5zqx37wnOE49mRl/+OtkIKGO7fAE 6 | 7 | ASIAIOSFODNN7EXAMPLE 8 | 9 | AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA== 10 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2021-11-09 15:14:59 +0000 (Tue, 09 Nov 2021) 4 | # 5 | # vim:ts=4:sts=4:sw=4:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | # https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners 17 | 18 | # Good in theory, to alert on PR changes to these code paths, but for public repos which may be forked and run .github/workflows/fork-update.yaml, this will result in a lot of spam 19 | 20 | # Tips: 21 | # 22 | # * includes changes under .github/ 23 | # dir/* only matches first level file changes but doesn't recurse 24 | # dir/ recurses 25 | # 26 | # - CODEOWNERS in base branch of PR determines review request 27 | # - paths are case sensitive 28 | # - last match wins, use * at top for overall owner then override with more specific teams 29 | 30 | #* @harisekhon # username or email address 31 | #* @myorg/platform-engineering # team based is the way to go - team must have Write access to the repo regardless of if individuals have access 32 | #* @myorg/devops 33 | #k8s @myorg/devops @myorg/sre-team 34 | #apps/ @myorg/developers 35 | #apps/dir2 # ignores dir2 as no owner/team specified on this line 36 | #src/ @myorg/developers 37 | #docs/ docs@example.com 38 | #.github/workflows @ci-cd-team 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Please be specific about your issue and include debug output from running with `-v -v -v` or for shell scripts after setting `export DEBUG=1` in your shell. 2 | 3 | You can anonymize hostnames / FQDNs, IP / MAC addresses, Kerberos principals, email addresses and almost anything else using `anonymize.pl` or the newer `anonymize.py` available in the [DevOps Perl Tools](https://github.com/HariSekhon/DevOps-Perl-tools) and [DevOps Python Tools](https://github.com/HariSekhon/DevOps-Python-tools) respectively. 4 | -------------------------------------------------------------------------------- /.github/workflows/centos.yaml.disabled: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: Tue Feb 4 09:53:28 2020 +0000 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | --- 15 | name: CentOS 16 | 17 | on: 18 | push: 19 | branches: 20 | - master 21 | paths-ignore: 22 | - '**/*.md' 23 | - '.github/workflows/*' 24 | - '!.github/workflows/centos.yaml' 25 | - '**/Dockerfile' 26 | - '**/Jenkinsfile' 27 | - '**/.envrc*' 28 | - .checkov.yaml 29 | - .dockerignore 30 | - .gcloudignore 31 | - .editorconfig 32 | - '.gitconfig*' 33 | - .gitignore 34 | - .grype.yaml 35 | - .hound.yml 36 | - .terraformignore 37 | - Jenkinsfile 38 | - .appveyor.yml 39 | - .buildkite/pipeline.yml 40 | - .circleci/config.yml 41 | - .cirrus.yml 42 | - .concourse.yml 43 | - .drone.yml 44 | - .gitlab-ci.yml 45 | - .gocd.yml 46 | - .scrutinizer.yml 47 | - .semaphore/semaphore.yml 48 | - .travis.yml 49 | - .werckerignore 50 | - azure-pipelines.yml 51 | - bitbucket-pipelines.yml 52 | - buddy.yml 53 | - buildspec.yml 54 | - cloudbuild.yaml 55 | - codefresh.yml 56 | - codeship.yml 57 | - shippable.yml 58 | - wercker.yml 59 | workflow_dispatch: 60 | inputs: 61 | debug: 62 | type: boolean 63 | required: false 64 | default: false 65 | schedule: 66 | - cron: '0 7 * * *' 67 | 68 | permissions: 69 | contents: read 70 | 71 | concurrency: 72 | group: ${{ github.workflow }}-${{ github.ref }} 73 | cancel-in-progress: true 74 | 75 | jobs: 76 | build: 77 | # github.event.repository context not available in scheduled workflows 78 | #if: github.event.repository.fork == false 79 | if: github.repository_owner == 'HariSekhon' 80 | name: Build 81 | uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master 82 | with: 83 | container: centos:latest 84 | caches: yum pip cpanm 85 | debug: ${{ github.event.inputs.debug }} 86 | -------------------------------------------------------------------------------- /.github/workflows/centos7.yaml.disabled: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: Tue Feb 4 09:53:28 2020 +0000 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | --- 15 | name: CentOS 7 16 | 17 | on: 18 | push: 19 | branches: 20 | - master 21 | paths-ignore: 22 | - '**/*.md' 23 | - '.github/workflows/*' 24 | - '!.github/workflows/centos7.yaml' 25 | - '**/Dockerfile' 26 | - '**/Jenkinsfile' 27 | - '**/.envrc*' 28 | - .checkov.yaml 29 | - .dockerignore 30 | - .gcloudignore 31 | - .editorconfig 32 | - '.gitconfig*' 33 | - .gitignore 34 | - .grype.yaml 35 | - .hound.yml 36 | - .terraformignore 37 | - Jenkinsfile 38 | - .appveyor.yml 39 | - .buildkite/pipeline.yml 40 | - .circleci/config.yml 41 | - .cirrus.yml 42 | - .concourse.yml 43 | - .drone.yml 44 | - .gitlab-ci.yml 45 | - .gocd.yml 46 | - .scrutinizer.yml 47 | - .semaphore/semaphore.yml 48 | - .travis.yml 49 | - .werckerignore 50 | - azure-pipelines.yml 51 | - bitbucket-pipelines.yml 52 | - buddy.yml 53 | - buildspec.yml 54 | - cloudbuild.yaml 55 | - codefresh.yml 56 | - codeship.yml 57 | - shippable.yml 58 | - wercker.yml 59 | workflow_dispatch: 60 | inputs: 61 | debug: 62 | type: boolean 63 | required: false 64 | default: false 65 | schedule: 66 | - cron: '0 7 * * *' 67 | 68 | permissions: 69 | contents: read 70 | 71 | concurrency: 72 | group: ${{ github.workflow }}-${{ github.ref }} 73 | cancel-in-progress: true 74 | 75 | jobs: 76 | build: 77 | # github.event.repository context not available in scheduled workflows 78 | #if: github.event.repository.fork == false 79 | if: github.repository_owner == 'HariSekhon' 80 | name: Build 81 | uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master 82 | with: 83 | container: centos:7 84 | caches: yum pip cpanm 85 | debug: ${{ github.event.inputs.debug }} 86 | -------------------------------------------------------------------------------- /.github/workflows/centos8.yaml.disabled: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: Tue Feb 4 09:53:28 2020 +0000 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | --- 15 | name: CentOS 8 16 | 17 | on: 18 | push: 19 | branches: 20 | - master 21 | paths-ignore: 22 | - '**/*.md' 23 | - '.github/workflows/*' 24 | - '!.github/workflows/centos8.yaml' 25 | - '**/Dockerfile' 26 | - '**/Jenkinsfile' 27 | - '**/.envrc*' 28 | - .checkov.yaml 29 | - .dockerignore 30 | - .gcloudignore 31 | - .editorconfig 32 | - '.gitconfig*' 33 | - .gitignore 34 | - .grype.yaml 35 | - .hound.yml 36 | - .terraformignore 37 | - Jenkinsfile 38 | - .appveyor.yml 39 | - .buildkite/pipeline.yml 40 | - .circleci/config.yml 41 | - .cirrus.yml 42 | - .concourse.yml 43 | - .drone.yml 44 | - .gitlab-ci.yml 45 | - .gocd.yml 46 | - .scrutinizer.yml 47 | - .semaphore/semaphore.yml 48 | - .travis.yml 49 | - .werckerignore 50 | - azure-pipelines.yml 51 | - bitbucket-pipelines.yml 52 | - buddy.yml 53 | - buildspec.yml 54 | - cloudbuild.yaml 55 | - codefresh.yml 56 | - codeship.yml 57 | - shippable.yml 58 | - wercker.yml 59 | workflow_dispatch: 60 | inputs: 61 | debug: 62 | type: boolean 63 | required: false 64 | default: false 65 | schedule: 66 | - cron: '0 7 * * *' 67 | 68 | permissions: 69 | contents: read 70 | 71 | concurrency: 72 | group: ${{ github.workflow }}-${{ github.ref }} 73 | cancel-in-progress: true 74 | 75 | jobs: 76 | build: 77 | # github.event.repository context not available in scheduled workflows 78 | #if: github.event.repository.fork == false 79 | if: github.repository_owner == 'HariSekhon' 80 | name: Build 81 | uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master 82 | with: 83 | container: centos:8 84 | caches: yum pip cpanm 85 | debug: ${{ github.event.inputs.debug }} 86 | -------------------------------------------------------------------------------- /.github/workflows/checkov.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: Tue Feb 4 09:53:28 2020 +0000 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | # ============================================================================ # 15 | # C h e c k o v G i t H u b W o r k f l o w 16 | # ============================================================================ # 17 | 18 | # Static analysis of Terraform code - publishes report to GitHub Security tab 19 | 20 | # https://github.com/bridgecrewio/checkov-action 21 | 22 | --- 23 | name: Checkov 24 | 25 | on: # yamllint disable-line rule:truthy 26 | push: 27 | branches: 28 | - master 29 | - main 30 | paths-ignore: 31 | - '**/*.md' 32 | pull_request: 33 | branches: 34 | - master 35 | - main 36 | paths-ignore: 37 | - '**/*.md' 38 | workflow_dispatch: 39 | inputs: 40 | debug: 41 | type: boolean 42 | required: false 43 | default: false 44 | schedule: 45 | - cron: '0 0 * * 1' 46 | 47 | permissions: 48 | actions: read 49 | contents: read 50 | security-events: write 51 | 52 | jobs: 53 | checkov: 54 | # github.event.repository context not available in scheduled workflows 55 | #if: github.event.repository.fork == false 56 | if: github.repository_owner == 'HariSekhon' 57 | name: Checkov 58 | uses: HariSekhon/GitHub-Actions/.github/workflows/checkov.yaml@master 59 | with: 60 | debug: ${{ github.event.inputs.debug }} 61 | -------------------------------------------------------------------------------- /.github/workflows/codeowners.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: Tue Feb 4 09:53:28 2020 +0000 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | # ============================================================================ # 15 | # C o d e O w n e r s 16 | # ============================================================================ # 17 | 18 | --- 19 | name: CodeOwners 20 | 21 | on: # yamllint disable-line rule:truthy 22 | push: 23 | branches: 24 | - master 25 | - main 26 | paths: 27 | - CODEOWNERS 28 | - .github/CODEOWNERS 29 | pull_request: 30 | branches: 31 | - master 32 | - main 33 | paths: 34 | - CODEOWNERS 35 | - .github/CODEOWNERS 36 | workflow_dispatch: 37 | inputs: 38 | debug: 39 | type: boolean 40 | required: false 41 | default: false 42 | schedule: 43 | - cron: '0 0 * * 1' 44 | 45 | permissions: 46 | contents: read 47 | 48 | concurrency: 49 | group: ${{ github.workflow }}-${{ github.ref }} 50 | cancel-in-progress: true 51 | 52 | jobs: 53 | validate: 54 | # github.event.repository context not available in scheduled workflows 55 | #if: github.event.repository.fork == false 56 | if: github.repository_owner == 'HariSekhon' 57 | name: Validate CODEOWNERS 58 | uses: HariSekhon/GitHub-Actions/.github/workflows/codeowners.yaml@master 59 | with: 60 | debug: ${{ github.event.inputs.debug }} 61 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "CodeQL" 3 | 4 | on: 5 | push: 6 | branches: 7 | - master 8 | pull_request: 9 | # The branches below must be a subset of the branches above 10 | branches: 11 | - master 12 | schedule: 13 | - cron: '37 15 * * 4' 14 | 15 | jobs: 16 | analyze: 17 | name: Analyze 18 | runs-on: ubuntu-latest 19 | permissions: 20 | actions: read 21 | contents: read 22 | security-events: write 23 | 24 | strategy: 25 | fail-fast: false 26 | matrix: 27 | language: 28 | - python 29 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 30 | # Learn more about CodeQL language support at https://git.io/codeql-language-support 31 | 32 | steps: 33 | - name: Checkout repository 34 | uses: actions/checkout@v2 35 | 36 | # Initializes the CodeQL tools for scanning. 37 | - name: Initialize CodeQL 38 | uses: github/codeql-action/init@v1 39 | with: 40 | languages: ${{ matrix.language }} 41 | # If you wish to specify custom queries, you can do so here or in a config file. 42 | # By default, queries listed here will override any specified in a config file. 43 | # Prefix the list here with "+" to use these queries and those in the config file. 44 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 45 | 46 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 47 | # If this step fails, then you should remove it and run the build manually (see below) 48 | - name: Autobuild 49 | uses: github/codeql-action/autobuild@v1 50 | 51 | # Command-line programs to run using the OS shell. 52 | # https://git.io/JvXDl 53 | 54 | # If the Autobuild fails above, remove it and uncomment the following three lines 55 | # and modify them (or add more) to build your code if your project 56 | # uses a compiled language 57 | 58 | #- run: | 59 | # make bootstrap 60 | # make release 61 | 62 | - name: Perform CodeQL Analysis 63 | uses: github/codeql-action/analyze@v1 64 | -------------------------------------------------------------------------------- /.github/workflows/debian_6.yaml.disabled: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: Tue Feb 4 09:53:28 2020 +0000 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | --- 15 | name: Debian 6 16 | 17 | on: 18 | push: 19 | branches: 20 | - master 21 | paths-ignore: 22 | - '**/*.md' 23 | workflow_dispatch: 24 | inputs: 25 | debug: 26 | type: boolean 27 | required: false 28 | default: false 29 | schedule: 30 | - cron: '0 7 * * *' 31 | 32 | permissions: 33 | contents: read 34 | 35 | concurrency: 36 | group: ${{ github.workflow }}-${{ github.ref }} 37 | cancel-in-progress: true 38 | 39 | jobs: 40 | build: 41 | # github.event.repository context not available in scheduled workflows 42 | #if: github.event.repository.fork == false 43 | if: github.repository_owner == 'HariSekhon' 44 | name: Build 45 | uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master 46 | with: 47 | container: debian:6 48 | # causes nodejs errors 49 | #caches: apt pip cpanm 50 | debug: ${{ github.event.inputs.debug }} 51 | -------------------------------------------------------------------------------- /.github/workflows/debian_7.yaml.disabled: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: Tue Feb 4 09:53:28 2020 +0000 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | --- 15 | name: Debian 7 16 | 17 | on: 18 | push: 19 | branches: 20 | - master 21 | paths-ignore: 22 | - '**/*.md' 23 | workflow_dispatch: 24 | inputs: 25 | debug: 26 | type: boolean 27 | required: false 28 | default: false 29 | schedule: 30 | - cron: '0 7 * * *' 31 | 32 | permissions: 33 | contents: read 34 | 35 | concurrency: 36 | group: ${{ github.workflow }}-${{ github.ref }} 37 | cancel-in-progress: true 38 | 39 | jobs: 40 | build: 41 | # github.event.repository context not available in scheduled workflows 42 | #if: github.event.repository.fork == false 43 | if: github.repository_owner == 'HariSekhon' 44 | name: Build 45 | uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master 46 | with: 47 | container: debian:7 48 | # causes nodejs errors 49 | #caches: apt pip cpanm 50 | debug: ${{ github.event.inputs.debug }} 51 | -------------------------------------------------------------------------------- /.github/workflows/debian_8.yaml.disabled: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: Tue Feb 4 09:53:28 2020 +0000 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | --- 15 | name: Debian 8 16 | 17 | on: 18 | push: 19 | branches: 20 | - master 21 | paths-ignore: 22 | - '**/*.md' 23 | - '.github/workflows/*' 24 | - '!.github/workflows/debian_8.yaml' 25 | - '**/Dockerfile' 26 | - '**/Jenkinsfile' 27 | - '**/.envrc*' 28 | - .checkov.yaml 29 | - .dockerignore 30 | - .gcloudignore 31 | - .editorconfig 32 | - '.gitconfig*' 33 | - .gitignore 34 | - .grype.yaml 35 | - .hound.yml 36 | - .terraformignore 37 | - Jenkinsfile 38 | - .appveyor.yml 39 | - .buildkite/pipeline.yml 40 | - .circleci/config.yml 41 | - .cirrus.yml 42 | - .concourse.yml 43 | - .drone.yml 44 | - .gitlab-ci.yml 45 | - .gocd.yml 46 | - .scrutinizer.yml 47 | - .semaphore/semaphore.yml 48 | - .travis.yml 49 | - .werckerignore 50 | - azure-pipelines.yml 51 | - bitbucket-pipelines.yml 52 | - buddy.yml 53 | - buildspec.yml 54 | - cloudbuild.yaml 55 | - codefresh.yml 56 | - codeship.yml 57 | - shippable.yml 58 | - wercker.yml 59 | workflow_dispatch: 60 | inputs: 61 | debug: 62 | type: boolean 63 | required: false 64 | default: false 65 | schedule: 66 | - cron: '0 7 * * *' 67 | 68 | permissions: 69 | contents: read 70 | 71 | concurrency: 72 | group: ${{ github.workflow }}-${{ github.ref }} 73 | cancel-in-progress: true 74 | 75 | jobs: 76 | build: 77 | # github.event.repository context not available in scheduled workflows 78 | #if: github.event.repository.fork == false 79 | if: github.repository_owner == 'HariSekhon' 80 | name: Build 81 | uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master 82 | with: 83 | container: debian:8 84 | caches: apt pip cpanm 85 | debug: ${{ github.event.inputs.debug }} 86 | -------------------------------------------------------------------------------- /.github/workflows/fork-sync.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: Tue Feb 4 09:53:28 2020 +0000 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | # ============================================================================ # 15 | # F o r k S y n c 16 | # ============================================================================ # 17 | 18 | # For a fork of the original repo, activate to keep it up to date via straight GitHub sync to the default branch 19 | 20 | --- 21 | name: Fork Sync 22 | 23 | on: # yamllint disable-line rule:truthy 24 | workflow_dispatch: 25 | inputs: 26 | debug: 27 | type: boolean 28 | required: false 29 | default: false 30 | schedule: 31 | - cron: '0 */3 * * *' 32 | 33 | permissions: 34 | contents: write 35 | 36 | concurrency: 37 | group: ${{ github.workflow }}-${{ github.ref }} 38 | cancel-in-progress: false 39 | 40 | jobs: 41 | fork_sync: 42 | # github.event.repository context not available in scheduled workflows 43 | #if: github.event.repository.fork == true 44 | if: github.repository_owner != 'HariSekhon' 45 | name: Fork Sync 46 | uses: HariSekhon/GitHub-Actions/.github/workflows/fork-sync.yaml@master 47 | with: 48 | debug: ${{ github.event.inputs.debug }} 49 | -------------------------------------------------------------------------------- /.github/workflows/fork-update-pr.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: Tue Feb 4 09:53:28 2020 +0000 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | # ============================================================================ # 15 | # F o r k U p d a t e P R 16 | # ============================================================================ # 17 | 18 | # For a fork of the original repo, activate to keep its branches up to date via Pull Requests 19 | # 20 | # To be used in conjunction with the adjacent fork-sync.yaml which keeps the default branch up to date 21 | 22 | --- 23 | name: Fork Update PR 24 | 25 | on: # yamllint disable-line rule:truthy 26 | workflow_dispatch: 27 | inputs: 28 | debug: 29 | type: boolean 30 | required: false 31 | default: false 32 | schedule: 33 | - cron: '0 10 * * 1' 34 | 35 | permissions: 36 | contents: write 37 | pull-requests: write 38 | 39 | concurrency: 40 | group: ${{ github.workflow }}-${{ github.ref }} 41 | cancel-in-progress: false 42 | 43 | jobs: 44 | fork_update_pr: 45 | # github.event.repository context not available in scheduled workflows 46 | #if: github.event.repository.fork == true 47 | if: github.repository_owner != 'HariSekhon' 48 | name: Fork Update PR 49 | uses: HariSekhon/GitHub-Actions/.github/workflows/fork-update-pr.yaml@master 50 | with: 51 | debug: ${{ github.event.inputs.debug }} 52 | -------------------------------------------------------------------------------- /.github/workflows/ghcr_python_ubuntu.yaml.disabled: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2022-02-09 18:07:10 +0000 (Wed, 09 Feb 2022) 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | --- 15 | name: GHCR Build (Ubuntu) 16 | 17 | on: 18 | push: 19 | branches: 20 | - master 21 | - main 22 | workflow_dispatch: 23 | 24 | jobs: 25 | docker_build: 26 | name: Docker Build 27 | uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build_ghcr.yaml@master 28 | with: 29 | image: pytools 30 | tags: ubuntu latest 31 | dockerfile-repo: HariSekhon/Dockerfiles 32 | context: Dockerfiles/devops-python-tools-ubuntu 33 | if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker' 34 | permissions: 35 | contents: read 36 | packages: write 37 | -------------------------------------------------------------------------------- /.github/workflows/grype.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2023-05-13 01:07:56 +0100 (Sat, 13 May 2023) 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | # ============================================================================ # 15 | # G r y p e 16 | # ============================================================================ # 17 | 18 | --- 19 | name: Grype 20 | 21 | on: # yamllint disable-line rule:truthy 22 | push: 23 | branches: 24 | - master 25 | - main 26 | paths-ignore: 27 | - '**/*.md' 28 | pull_request: 29 | branches: 30 | - master 31 | - main 32 | paths-ignore: 33 | - '**/*.md' 34 | workflow_dispatch: 35 | inputs: 36 | debug: 37 | type: boolean 38 | required: false 39 | default: false 40 | schedule: 41 | - cron: '0 0 * * 1' 42 | 43 | permissions: 44 | actions: read 45 | contents: read 46 | security-events: write 47 | 48 | concurrency: 49 | group: ${{ github.workflow }}-${{ github.ref }} 50 | cancel-in-progress: true 51 | 52 | jobs: 53 | Grype: 54 | # github.event.repository context not available in scheduled workflows 55 | #if: github.event.repository.fork == false 56 | if: github.repository_owner == 'HariSekhon' 57 | name: Grype 58 | uses: HariSekhon/GitHub-Actions/.github/workflows/grype.yaml@master 59 | with: 60 | debug: ${{ github.event.inputs.debug }} 61 | -------------------------------------------------------------------------------- /.github/workflows/json.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: Tue Feb 4 09:53:28 2020 +0000 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | # ============================================================================ # 15 | # J S O N 16 | # ============================================================================ # 17 | 18 | # Validate any JSON files found in the repo 19 | 20 | --- 21 | name: JSON 22 | 23 | on: # yamllint disable-line rule:truthy 24 | push: 25 | branches: 26 | - master 27 | - main 28 | paths: 29 | - '**/*.json' 30 | - .github/workflows/json.yaml 31 | pull_request: 32 | branches: 33 | - master 34 | - main 35 | paths: 36 | - '**/*.json' 37 | - .github/workflows/json.yaml 38 | workflow_dispatch: 39 | inputs: 40 | debug: 41 | type: boolean 42 | required: false 43 | default: false 44 | schedule: 45 | - cron: '0 0 * * 1' 46 | 47 | permissions: 48 | contents: read 49 | 50 | jobs: 51 | check_json: 52 | # github.event.repository context not available in scheduled workflows 53 | #if: github.event.repository.fork == false 54 | if: github.repository_owner == 'HariSekhon' 55 | name: Check JSON 56 | uses: HariSekhon/GitHub-Actions/.github/workflows/json.yaml@master 57 | with: 58 | debug: ${{ github.event.inputs.debug }} 59 | -------------------------------------------------------------------------------- /.github/workflows/kics.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2022-02-01 19:36:08 +0000 (Tue, 01 Feb 2022) 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | # ============================================================================ # 15 | # K i c s 16 | # ============================================================================ # 17 | 18 | --- 19 | name: Kics 20 | 21 | on: # yamllint disable-line rule:truthy 22 | push: 23 | branches: 24 | - master 25 | - main 26 | paths-ignore: 27 | - '**/*.md' 28 | pull_request: 29 | branches: 30 | - master 31 | - main 32 | paths-ignore: 33 | - '**/*.md' 34 | workflow_dispatch: 35 | inputs: 36 | debug: 37 | type: boolean 38 | required: false 39 | default: false 40 | schedule: 41 | - cron: '0 0 * * 1' 42 | 43 | permissions: 44 | actions: read 45 | contents: read 46 | security-events: write 47 | 48 | concurrency: 49 | group: ${{ github.workflow }}-${{ github.ref }} 50 | cancel-in-progress: true 51 | 52 | jobs: 53 | kics: 54 | # github.event.repository context not available in scheduled workflows 55 | #if: github.event.repository.fork == false 56 | if: github.repository_owner == 'HariSekhon' 57 | name: Kics 58 | uses: HariSekhon/GitHub-Actions/.github/workflows/kics.yaml@master 59 | with: 60 | debug: ${{ github.event.inputs.debug }} 61 | -------------------------------------------------------------------------------- /.github/workflows/markdown.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2023-04-14 23:53:43 +0100 (Fri, 14 Apr 2023) 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | # ============================================================================ # 15 | # M a r k D o w n 16 | # ============================================================================ # 17 | 18 | --- 19 | name: Markdown 20 | 21 | on: # yamllint disable-line rule:truthy 22 | push: 23 | branches: 24 | - master 25 | - main 26 | paths: 27 | - '**/*.md' 28 | - .mdlrc 29 | - .mdl.rb 30 | - .markdownlint.rb 31 | - .github/workflows/markdown.yaml 32 | pull_request: 33 | branches: 34 | - master 35 | - main 36 | paths: 37 | - '**/*.md' 38 | - .mdlrc 39 | - .mdl.rb 40 | - .markdownlint.rb 41 | - .github/workflows/markdown.yaml 42 | workflow_dispatch: 43 | 44 | permissions: 45 | contents: read 46 | pull-requests: read 47 | 48 | jobs: 49 | Markdown: 50 | # github.event.repository context not available in scheduled workflows 51 | #if: github.event.repository.fork == false 52 | if: github.repository_owner == 'HariSekhon' 53 | name: Markdown 54 | uses: HariSekhon/GitHub-Actions/.github/workflows/markdown.yaml@master 55 | -------------------------------------------------------------------------------- /.github/workflows/pypy2.yaml.disabled: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | --- 15 | name: PyPy 2 16 | 17 | on: 18 | push: 19 | branches: 20 | - master 21 | - main 22 | paths: 23 | - '**/*.py' 24 | pull_request: 25 | branches: 26 | - master 27 | - main 28 | paths: 29 | - '**/*.py' 30 | workflow_dispatch: 31 | inputs: 32 | debug: 33 | type: boolean 34 | required: false 35 | default: false 36 | schedule: 37 | - cron: '0 7 * * *' 38 | 39 | permissions: 40 | contents: read 41 | 42 | concurrency: 43 | group: ${{ github.workflow }}-${{ github.ref }} 44 | cancel-in-progress: true 45 | 46 | jobs: 47 | build: 48 | # github.event.repository context not available in scheduled workflows 49 | #if: github.event.repository.fork == false 50 | if: github.repository_owner == 'HariSekhon' 51 | name: PyPy2 52 | uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master 53 | with: 54 | python-version: pypy2 55 | caches: apt pip 56 | debug: ${{ github.event.inputs.debug }} 57 | -------------------------------------------------------------------------------- /.github/workflows/pypy3.yaml.disabled: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | --- 15 | name: PyPy 3 16 | 17 | on: 18 | push: 19 | branches: 20 | - master 21 | - main 22 | paths: 23 | - '**/*.py' 24 | pull_request: 25 | branches: 26 | - master 27 | - main 28 | paths: 29 | - '**/*.py' 30 | workflow_dispatch: 31 | inputs: 32 | debug: 33 | type: boolean 34 | required: false 35 | default: false 36 | schedule: 37 | - cron: '0 7 * * *' 38 | 39 | permissions: 40 | contents: read 41 | 42 | concurrency: 43 | group: ${{ github.workflow }}-${{ github.ref }} 44 | cancel-in-progress: true 45 | 46 | jobs: 47 | build: 48 | # github.event.repository context not available in scheduled workflows 49 | #if: github.event.repository.fork == false 50 | if: github.repository_owner == 'HariSekhon' 51 | name: PyPy3 52 | uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master 53 | with: 54 | python-version: pypy3 55 | caches: apt pip 56 | debug: ${{ github.event.inputs.debug }} 57 | -------------------------------------------------------------------------------- /.github/workflows/python2.7.yaml.disabled: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | --- 15 | name: Python 2.7 16 | 17 | on: 18 | push: 19 | branches: 20 | - master 21 | - main 22 | paths: 23 | - '**/*.py' 24 | - pylib 25 | - requirements.txt 26 | - .github/workflows/python2.7.yaml 27 | pull_request: 28 | branches: 29 | - master 30 | - main 31 | paths: 32 | - '**/*.py' 33 | - pylib 34 | - requirements.txt 35 | - .github/workflows/python2.7.yaml 36 | workflow_dispatch: 37 | inputs: 38 | debug: 39 | type: boolean 40 | required: false 41 | default: false 42 | schedule: 43 | - cron: '0 7 * * *' 44 | 45 | permissions: 46 | contents: read 47 | 48 | concurrency: 49 | group: ${{ github.workflow }}-${{ github.ref }} 50 | cancel-in-progress: true 51 | 52 | jobs: 53 | build: 54 | # github.event.repository context not available in scheduled workflows 55 | #if: github.event.repository.fork == false 56 | if: github.repository_owner == 'HariSekhon' 57 | name: Python 2.7 58 | uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master 59 | with: 60 | python-version: 2.7 61 | caches: apt pip 62 | debug: ${{ github.event.inputs.debug }} 63 | -------------------------------------------------------------------------------- /.github/workflows/python3.10.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | --- 15 | name: Python 3.10 16 | 17 | on: # yamllint disable-line rule:truthy 18 | push: 19 | branches: 20 | - master 21 | - main 22 | paths: 23 | - '**/*.py' 24 | - pylib 25 | - requirements.txt 26 | - .github/workflows/python3.10.yaml 27 | pull_request: 28 | branches: 29 | - master 30 | - main 31 | paths: 32 | - '**/*.py' 33 | - pylib 34 | - requirements.txt 35 | - .github/workflows/python3.10.yaml 36 | workflow_dispatch: 37 | inputs: 38 | debug: 39 | type: boolean 40 | required: false 41 | default: false 42 | schedule: 43 | - cron: '0 7 * * *' 44 | 45 | permissions: 46 | contents: read 47 | 48 | concurrency: 49 | group: ${{ github.workflow }}-${{ github.ref }} 50 | cancel-in-progress: true 51 | 52 | jobs: 53 | build: 54 | # github.event.repository context not available in scheduled workflows 55 | #if: github.event.repository.fork == false 56 | if: github.repository_owner == 'HariSekhon' 57 | name: Python 3.10 58 | uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master 59 | with: 60 | python-version: "3.10" 61 | caches: apt pip 62 | debug: ${{ github.event.inputs.debug }} 63 | -------------------------------------------------------------------------------- /.github/workflows/python3.11.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | --- 15 | name: Python 3.11 16 | 17 | on: # yamllint disable-line rule:truthy 18 | push: 19 | branches: 20 | - master 21 | - main 22 | paths: 23 | - '**/*.py' 24 | - pylib 25 | - requirements.txt 26 | - .github/workflows/python3.11.yaml 27 | pull_request: 28 | branches: 29 | - master 30 | - main 31 | paths: 32 | - '**/*.py' 33 | - pylib 34 | - requirements.txt 35 | - .github/workflows/python3.11.yaml 36 | workflow_dispatch: 37 | inputs: 38 | debug: 39 | type: boolean 40 | required: false 41 | default: false 42 | schedule: 43 | - cron: '0 7 * * *' 44 | 45 | permissions: 46 | contents: read 47 | 48 | concurrency: 49 | group: ${{ github.workflow }}-${{ github.ref }} 50 | cancel-in-progress: true 51 | 52 | jobs: 53 | build: 54 | # github.event.repository context not available in scheduled workflows 55 | #if: github.event.repository.fork == false 56 | if: github.repository_owner == 'HariSekhon' 57 | name: Python 3.11 58 | uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master 59 | with: 60 | python-version: "3.11" 61 | caches: apt pip 62 | debug: ${{ github.event.inputs.debug }} 63 | -------------------------------------------------------------------------------- /.github/workflows/python3.6.yaml.disabled: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | --- 15 | name: Python 3.6 16 | 17 | on: 18 | push: 19 | branches: 20 | - master 21 | - main 22 | paths: 23 | - '**/*.py' 24 | - pylib 25 | - requirements.txt 26 | - .github/workflows/python3.6.yaml 27 | pull_request: 28 | branches: 29 | - master 30 | - main 31 | paths: 32 | - '**/*.py' 33 | - pylib 34 | - requirements.txt 35 | - .github/workflows/python3.6.yaml 36 | workflow_dispatch: 37 | inputs: 38 | debug: 39 | type: boolean 40 | required: false 41 | default: false 42 | schedule: 43 | - cron: '0 7 * * *' 44 | 45 | permissions: 46 | contents: read 47 | 48 | concurrency: 49 | group: ${{ github.workflow }}-${{ github.ref }} 50 | cancel-in-progress: true 51 | 52 | jobs: 53 | build: 54 | # github.event.repository context not available in scheduled workflows 55 | #if: github.event.repository.fork == false 56 | if: github.repository_owner == 'HariSekhon' 57 | name: Python 3.6 58 | uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master 59 | with: 60 | python-version: 3.6 61 | caches: apt pip 62 | debug: ${{ github.event.inputs.debug }} 63 | -------------------------------------------------------------------------------- /.github/workflows/python3.7.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | --- 15 | name: Python 3.7 16 | 17 | on: # yamllint disable-line rule:truthy 18 | push: 19 | branches: 20 | - master 21 | - main 22 | paths: 23 | - '**/*.py' 24 | - pylib 25 | - requirements.txt 26 | - .github/workflows/python3.7.yaml 27 | pull_request: 28 | branches: 29 | - master 30 | - main 31 | paths: 32 | - '**/*.py' 33 | - pylib 34 | - requirements.txt 35 | - .github/workflows/python3.7.yaml 36 | workflow_dispatch: 37 | inputs: 38 | debug: 39 | type: boolean 40 | required: false 41 | default: false 42 | schedule: 43 | - cron: '0 7 * * *' 44 | 45 | permissions: 46 | contents: read 47 | 48 | concurrency: 49 | group: ${{ github.workflow }}-${{ github.ref }} 50 | cancel-in-progress: true 51 | 52 | jobs: 53 | build: 54 | # github.event.repository context not available in scheduled workflows 55 | #if: github.event.repository.fork == false 56 | if: github.repository_owner == 'HariSekhon' 57 | name: Python 3.7 58 | uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master 59 | with: 60 | python-version: 3.7 61 | caches: apt pip 62 | debug: ${{ github.event.inputs.debug }} 63 | -------------------------------------------------------------------------------- /.github/workflows/python3.8.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | --- 15 | name: Python 3.8 16 | 17 | on: # yamllint disable-line rule:truthy 18 | push: 19 | branches: 20 | - master 21 | - main 22 | paths: 23 | - '**/*.py' 24 | - pylib 25 | - requirements.txt 26 | - .github/workflows/python3.8.yaml 27 | pull_request: 28 | branches: 29 | - master 30 | - main 31 | paths: 32 | - '**/*.py' 33 | - pylib 34 | - requirements.txt 35 | - .github/workflows/python3.8.yaml 36 | workflow_dispatch: 37 | inputs: 38 | debug: 39 | type: boolean 40 | required: false 41 | default: false 42 | schedule: 43 | - cron: '0 7 * * *' 44 | 45 | permissions: 46 | contents: read 47 | 48 | concurrency: 49 | group: ${{ github.workflow }}-${{ github.ref }} 50 | cancel-in-progress: true 51 | 52 | jobs: 53 | build: 54 | # github.event.repository context not available in scheduled workflows 55 | #if: github.event.repository.fork == false 56 | if: github.repository_owner == 'HariSekhon' 57 | name: Python 3.8 58 | uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master 59 | with: 60 | python-version: 3.8 61 | caches: apt pip 62 | debug: ${{ github.event.inputs.debug }} 63 | -------------------------------------------------------------------------------- /.github/workflows/python3.9.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | --- 15 | name: Python 3.9 16 | 17 | on: # yamllint disable-line rule:truthy 18 | push: 19 | branches: 20 | - master 21 | - main 22 | paths: 23 | - '**/*.py' 24 | - pylib 25 | - requirements.txt 26 | - .github/workflows/python3.9.yaml 27 | pull_request: 28 | branches: 29 | - master 30 | - main 31 | paths: 32 | - '**/*.py' 33 | - pylib 34 | - requirements.txt 35 | - .github/workflows/python3.9.yaml 36 | workflow_dispatch: 37 | inputs: 38 | debug: 39 | type: boolean 40 | required: false 41 | default: false 42 | schedule: 43 | - cron: '0 7 * * *' 44 | 45 | permissions: 46 | contents: read 47 | 48 | concurrency: 49 | group: ${{ github.workflow }}-${{ github.ref }} 50 | cancel-in-progress: true 51 | 52 | jobs: 53 | build: 54 | # github.event.repository context not available in scheduled workflows 55 | #if: github.event.repository.fork == false 56 | if: github.repository_owner == 'HariSekhon' 57 | name: Python 3.9 58 | uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master 59 | with: 60 | python-version: 3.9 61 | caches: apt pip 62 | debug: ${{ github.event.inputs.debug }} 63 | -------------------------------------------------------------------------------- /.github/workflows/semgrep-cloud.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: Tue Feb 4 09:53:28 2020 +0000 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | # ============================================================================ # 15 | # S e m g r e p C l o u d W o r k f l o w 16 | # ============================================================================ # 17 | 18 | # Logs results to https://semgrep.dev/ 19 | 20 | --- 21 | name: Semgrep Cloud 22 | 23 | on: # yamllint disable-line rule:truthy 24 | push: 25 | branches: 26 | - master 27 | - main 28 | paths-ignore: 29 | - '**/*.md' 30 | pull_request: 31 | branches: 32 | - master 33 | - main 34 | paths-ignore: 35 | - '**/*.md' 36 | workflow_dispatch: 37 | inputs: 38 | debug: 39 | type: boolean 40 | required: false 41 | default: false 42 | schedule: 43 | - cron: '0 0 * * 1' 44 | 45 | permissions: 46 | contents: read 47 | 48 | concurrency: 49 | group: ${{ github.workflow }}-${{ github.ref }} 50 | cancel-in-progress: true 51 | 52 | jobs: 53 | semgrep: 54 | # github.event.repository context not available in scheduled workflows 55 | #if: github.event.repository.fork == false 56 | if: github.repository_owner == 'HariSekhon' 57 | name: Semgrep Cloud 58 | uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep-cloud.yaml@master 59 | secrets: 60 | SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }} 61 | with: 62 | debug: ${{ github.event.inputs.debug }} 63 | -------------------------------------------------------------------------------- /.github/workflows/semgrep.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: Tue Feb 4 09:53:28 2020 +0000 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | # ============================================================================ # 15 | # S e m g r e p G i t H u b W o r k f l o w 16 | # ============================================================================ # 17 | 18 | # Generates code scanning alerts in GitHub's Security tab -> Code scanning alerts 19 | 20 | # https://semgrep.dev/docs/semgrep-ci/sample-ci-configs/#github-actions 21 | 22 | --- 23 | name: Semgrep 24 | 25 | on: # yamllint disable-line rule:truthy 26 | push: 27 | branches: 28 | - master 29 | - main 30 | paths-ignore: 31 | - '**/*.md' 32 | pull_request: 33 | branches: 34 | - master 35 | - main 36 | paths-ignore: 37 | - '**/*.md' 38 | workflow_dispatch: 39 | inputs: 40 | debug: 41 | type: boolean 42 | required: false 43 | default: false 44 | schedule: 45 | - cron: '0 0 * * 1' 46 | 47 | permissions: 48 | actions: read 49 | contents: read 50 | security-events: write 51 | 52 | concurrency: 53 | group: ${{ github.workflow }}-${{ github.ref }} 54 | cancel-in-progress: true 55 | 56 | jobs: 57 | semgrep: 58 | # github.event.repository context not available in scheduled workflows 59 | #if: github.event.repository.fork == false 60 | if: github.repository_owner == 'HariSekhon' 61 | name: Semgrep GitHub Security Tab 62 | uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master 63 | with: 64 | debug: ${{ github.event.inputs.debug }} 65 | -------------------------------------------------------------------------------- /.github/workflows/shellcheck.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: Tue Feb 4 09:53:28 2020 +0000 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | # ============================================================================ # 15 | # S h e l l C h e c k 16 | # ============================================================================ # 17 | 18 | # Validate any shell scripts found in the repo 19 | 20 | --- 21 | name: ShellCheck 22 | 23 | on: # yamllint disable-line rule:truthy 24 | push: 25 | branches: 26 | - master 27 | - main 28 | paths: 29 | - '**/*.sh' 30 | - .github/workflows/shellcheck.yaml 31 | pull_request: 32 | branches: 33 | - master 34 | - main 35 | paths: 36 | - '**/*.sh' 37 | - .github/workflows/shellcheck.yaml 38 | workflow_dispatch: 39 | inputs: 40 | debug: 41 | type: boolean 42 | required: false 43 | default: false 44 | 45 | permissions: 46 | contents: read 47 | 48 | concurrency: 49 | group: ${{ github.workflow }}-${{ github.ref }} 50 | cancel-in-progress: true 51 | 52 | jobs: 53 | shellcheck: 54 | # github.event.repository context not available in scheduled workflows 55 | #if: github.event.repository.fork == false 56 | if: github.repository_owner == 'HariSekhon' 57 | name: ShellCheck 58 | uses: HariSekhon/GitHub-Actions/.github/workflows/shellcheck.yaml@master 59 | with: 60 | debug: ${{ github.event.inputs.debug }} 61 | -------------------------------------------------------------------------------- /.github/workflows/trivy.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2022-02-02 11:27:37 +0000 (Wed, 02 Feb 2022) 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | # ============================================================================ # 15 | # T r i v y 16 | # ============================================================================ # 17 | 18 | # Scan files in the local repo 19 | 20 | --- 21 | name: Trivy 22 | 23 | on: # yamllint disable-line rule:truthy 24 | push: 25 | branches: 26 | - master 27 | - main 28 | paths-ignore: 29 | - '**/*.md' 30 | pull_request: 31 | branches: 32 | - master 33 | - main 34 | paths-ignore: 35 | - '**/*.md' 36 | workflow_dispatch: 37 | inputs: 38 | debug: 39 | type: boolean 40 | required: false 41 | default: false 42 | schedule: 43 | - cron: '0 0 * * 1' 44 | 45 | permissions: 46 | actions: read 47 | contents: read 48 | security-events: write 49 | 50 | concurrency: 51 | group: ${{ github.workflow }}-${{ github.ref }} 52 | cancel-in-progress: true 53 | 54 | jobs: 55 | trivy: 56 | # github.event.repository context not available in scheduled workflows 57 | #if: github.event.repository.fork == false 58 | if: github.repository_owner == 'HariSekhon' 59 | name: Trivy 60 | uses: HariSekhon/GitHub-Actions/.github/workflows/trivy.yaml@master 61 | with: 62 | debug: ${{ github.event.inputs.debug }} 63 | -------------------------------------------------------------------------------- /.github/workflows/validate.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: Tue Feb 4 09:53:28 2020 +0000 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | # ============================================================================ # 15 | # V a l i d a t i o n 16 | # ============================================================================ # 17 | 18 | # Run all custom validations against files in the repo 19 | 20 | --- 21 | name: Validation 22 | 23 | on: # yamllint disable-line rule:truthy 24 | push: 25 | branches: 26 | - master 27 | - main 28 | paths-ignore: 29 | - '**/*.md' 30 | pull_request: 31 | branches: 32 | - master 33 | - main 34 | paths-ignore: 35 | - '**/*.md' 36 | workflow_dispatch: 37 | inputs: 38 | debug: 39 | type: boolean 40 | required: false 41 | default: false 42 | #schedule: 43 | # - cron: '0 0 * * 1' 44 | 45 | permissions: 46 | contents: read 47 | 48 | concurrency: 49 | group: ${{ github.workflow }}-${{ github.ref }} 50 | cancel-in-progress: true 51 | 52 | jobs: 53 | validate: 54 | # github.event.repository context not available in scheduled workflows 55 | #if: github.event.repository.fork == false 56 | if: github.repository_owner == 'HariSekhon' 57 | name: Validate 58 | uses: HariSekhon/GitHub-Actions/.github/workflows/validate.yaml@master 59 | with: 60 | debug: ${{ github.event.inputs.debug }} 61 | -------------------------------------------------------------------------------- /.github/workflows/xml.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: Tue Feb 4 09:53:28 2020 +0000 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | # ============================================================================ # 15 | # X M L 16 | # ============================================================================ # 17 | 18 | # Validate any XML files found in the repo 19 | 20 | --- 21 | name: XML 22 | 23 | on: # yamllint disable-line rule:truthy 24 | push: 25 | branches: 26 | - master 27 | - main 28 | paths: 29 | - '**/*.xml' 30 | - .github/workflows/xml.yaml 31 | pull_request: 32 | branches: 33 | - master 34 | - main 35 | paths: 36 | - '**/*.xml' 37 | - .github/workflows/xml.yaml 38 | workflow_dispatch: 39 | inputs: 40 | debug: 41 | type: boolean 42 | required: false 43 | default: false 44 | schedule: 45 | - cron: '0 0 * * 1' 46 | 47 | permissions: 48 | contents: read 49 | 50 | jobs: 51 | check_xml: 52 | # github.event.repository context not available in scheduled workflows 53 | #if: github.event.repository.fork == false 54 | if: github.repository_owner == 'HariSekhon' 55 | name: Check XML 56 | uses: HariSekhon/GitHub-Actions/.github/workflows/xml.yaml@master 57 | -------------------------------------------------------------------------------- /.github/workflows/yaml.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: Tue Feb 4 09:53:28 2020 +0000 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | # ============================================================================ # 15 | # Y A M L 16 | # ============================================================================ # 17 | 18 | # Validate any YAML files found in the repo 19 | 20 | --- 21 | name: YAML 22 | 23 | on: # yamllint disable-line rule:truthy 24 | push: 25 | branches: 26 | - master 27 | - main 28 | paths: 29 | - '**/*.yml' 30 | - '**/*.yaml' 31 | - .github/workflows/yaml.yaml 32 | pull_request: 33 | branches: 34 | - master 35 | - main 36 | paths: 37 | - '**/*.yml' 38 | - '**/*.yaml' 39 | - .github/workflows/yaml.yaml 40 | workflow_dispatch: 41 | inputs: 42 | debug: 43 | type: boolean 44 | required: false 45 | default: false 46 | schedule: 47 | - cron: '0 0 * * 1' 48 | 49 | permissions: 50 | contents: read 51 | 52 | concurrency: 53 | group: ${{ github.workflow }}-${{ github.ref }} 54 | cancel-in-progress: true 55 | 56 | jobs: 57 | check_yaml: 58 | # github.event.repository context not available in scheduled workflows 59 | #if: github.event.repository.fork == false 60 | if: github.repository_owner == 'HariSekhon' 61 | name: Check YAML 62 | uses: HariSekhon/GitHub-Actions/.github/workflows/yaml.yaml@master 63 | with: 64 | debug: ${{ github.event.inputs.debug }} 65 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | # vim:ts=2:sts=2:sw=2:et 2 | # 3 | # Author: Hari Sekhon 4 | # Date: Sun Feb 23 19:02:10 2020 +0000 5 | # 6 | # https://github.com/HariSekhon/DevOps-Python-tools 7 | # 8 | # License: see accompanying Hari Sekhon LICENSE file 9 | # 10 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 11 | # to help improve or steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | # ============================================================================ # 17 | # G i t L a b C I 18 | # ============================================================================ # 19 | 20 | # https://docs.gitlab.com/ee/ci/yaml/README.html 21 | 22 | #include: '.gitlab/*.y*ml' 23 | 24 | image: ubuntu:18.04 25 | 26 | job: 27 | before_script: 28 | - setup/ci_bootstrap.sh 29 | script: 30 | - make init && make ci test 31 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "pylib"] 2 | path = pylib 3 | url = https://github.com/HariSekhon/pylib 4 | branch = master 5 | [submodule "bash-tools"] 6 | path = bash-tools 7 | url = https://github.com/HariSekhon/DevOps-Bash-tools 8 | branch = master 9 | [submodule "sql"] 10 | path = sql 11 | url = https://github.com/HariSekhon/SQL-scripts 12 | branch = master 13 | [submodule "templates"] 14 | path = templates 15 | url = https://github.com/HariSekhon/Templates 16 | branch = master 17 | -------------------------------------------------------------------------------- /.ipython-notebook-pyspark.00-pyspark-setup.py: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 22/8/2014 4 | # 5 | 6 | # Yarn library support 7 | 8 | # Requires SPARK_HOME to be set 9 | 10 | __author__ = "Hari Sekhon" 11 | __version__ = "0.1" 12 | 13 | import glob 14 | import os 15 | import sys 16 | 17 | # This only runs PySpark in local mode, not Yarn mode 18 | # 19 | # See ipython-notebook-spark.py for cluster mode (YARN or Standalone) 20 | 21 | spark_home = os.getenv('SPARK_HOME', None) 22 | if not spark_home: 23 | raise ValueError('SPARK_HOME environment variable is not set') 24 | sys.path.insert(0, os.path.join(spark_home, 'python')) 25 | for lib in glob.glob(os.path.join(spark_home, 'python/lib/py4j-*-src.zip')): 26 | sys.path.insert(0, lib) 27 | execfile(os.path.join(spark_home, 'python/pyspark/shell.py')) # pylint: disable=undefined-variable 28 | -------------------------------------------------------------------------------- /.ipython-notebook-pyspark.ipython_notebook_config.py.j2: -------------------------------------------------------------------------------- 1 | # 2 | # Generated by Hari Sekhon's {{ name }} on {{ date }} 3 | # 4 | # Can be hand-edited as long as you leave the path to the passwd.txt file, 5 | # otherwise will be overwritten on next run of {{ name }} 6 | # 7 | # Template Source: {{ template_path }} 8 | # 9 | 10 | c = get_config() 11 | # Set to 0.0.0.0 for portability but better set to your IP 12 | # for feedback to given users the correct URL 13 | c.NotebookApp.ip = '{{ ip }}' 14 | #c.NotebookApp.password = u'{{ password }}' 15 | passwd_txt = '{{ passwd_txt }}' 16 | c.NotebookApp.password = open(passwd_txt).read().strip() 17 | c.NotebookApp.open_browser = False 18 | c.IPKernelApp.pylab = 'inline' 19 | -------------------------------------------------------------------------------- /.mdl.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # vim:ts=4:sts=4:sw=4:et:filetype=ruby 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2024-08-22 01:58:12 +0200 (Thu, 22 Aug 2024) 6 | # 7 | # https///github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | all 17 | #exclude_rule 'MD001' 18 | #exclude_rule 'MD003' 19 | #exclude_rule 'MD005' 20 | exclude_rule 'MD007' # leave 2 space indentation for lists, 3 space is ugly af 21 | #exclude_rule 'MD012' 22 | exclude_rule 'MD013' # long lines cannot be split if they are URLs 23 | #exclude_rule 'MD022' 24 | #exclude_rule 'MD025' 25 | #exclude_rule 'MD031' 26 | #exclude_rule 'MD032' 27 | exclude_rule 'MD033' # inline HTML is important for formatting 28 | exclude_rule 'MD036' # emphasis used instead of header for footer Ported from lines 29 | #exclude_rule 'MD039' 30 | #exclude_rule 'MD056' 31 | -------------------------------------------------------------------------------- /.mdlrc: -------------------------------------------------------------------------------- 1 | mdlrc_dir = File.expand_path('..', __FILE__) 2 | 3 | style_file = File.join(mdlrc_dir, '.mdl.rb') 4 | 5 | style style_file 6 | -------------------------------------------------------------------------------- /.pyup.yml: -------------------------------------------------------------------------------- 1 | # https://pyup.io/docs/configuration/ 2 | 3 | update: insecure 4 | assignees: HariSekhon 5 | -------------------------------------------------------------------------------- /.sonarcloud.properties: -------------------------------------------------------------------------------- 1 | sonar.host.url=https://sonarcloud.io 2 | -------------------------------------------------------------------------------- /.yamllint: -------------------------------------------------------------------------------- 1 | # vim:ts=4:sts=4:sw=4:et 2 | # 3 | # Author: Hari Sekhon 4 | # Date: 2019-02-26 14:42:07 +0000 (Tue, 26 Feb 2019) 5 | # 6 | # https://github.com/HariSekhon/DevOps-Python-tools 7 | # 8 | # License: see accompanying Hari Sekhon LICENSE file 9 | # 10 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 11 | # 12 | # https://www.linkedin.com/in/HariSekhon 13 | # 14 | 15 | --- 16 | 17 | extends: default 18 | 19 | rules: 20 | # error too many spaces after colon 21 | colons: disable 22 | 23 | # warning missing starting space in comment 24 | comments: disable 25 | 26 | # warning comment not indented like content 27 | comments-indentation: disable 28 | 29 | # warning missing document start "---" 30 | document-start: disable 31 | 32 | # error line too long (94 > 80 characters) 33 | line-length: disable 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2015 Hari Sekhon 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /ambari_blueprints/README.md: -------------------------------------------------------------------------------- 1 | Hari Sekhon's Ambari Blueprint Samples 2 | ====================================== 3 | 4 | ### PyTools ```ambari_blueprints.py``` templates ### 5 | 6 | Generated by the ```ambari_blueprints.py``` tool found in the top level of this repo from various generic test clusters. 7 | 8 | Beware the default password is set to ```test``` - must modify ```hostmappings*.json``` to set a real password. 9 | 10 | Also note that the high degree of variation in cluster topologies when combined with the large number of components especially in multi-HA deployments makes this less portable than you would expect and should not be taken as-is for production - you should hire an experienced Hadoop expert. 11 | 12 | Ambari Blueprints are very raw and there are lots of things that can go wrong with them even if they submit correctly to Ambari they can fail half way through cluster deployment, often due to bugs such as [AMBARI-14569](https://issues.apache.org/jira/browse/AMBARI-14569). There are also thousands of lines of optional json configuration settings that may need to be added/tweaked, which have been omitted here to try to keep things as a generic base. 13 | 14 | Some features will not work in different versions of Ambari < 2.1.0 (HA, host predicates/counts) or require extra configuration eg. for various services HA deployments. You must read all the Ambari documentation thoroughly. 15 | 16 | ##### I am not supporting these Blueprints, please do not raise issues for this part of the repo unless you're submitting fixes/improvements as I likely won't have the time or infrastructure to debug it for you. ##### 17 | 18 | This is only for my own usage and reference for modification to my clients needs while consulting. 19 | -------------------------------------------------------------------------------- /ambari_blueprints/hostmappings-hdfs-ha-random-hosts.json: -------------------------------------------------------------------------------- 1 | { 2 | "blueprint": "cluster-ha-named", 3 | "default_password": "test", 4 | "host_groups": [ 5 | { 6 | "name": "NN1", 7 | "hosts": [ 8 | { 9 | "fqdn": "host1.internal" 10 | } 11 | ] 12 | }, 13 | { 14 | "name": "NN2", 15 | "host_count": 1 16 | }, 17 | { 18 | "name": "QJM3", 19 | "host_count": 1 20 | }, 21 | { 22 | "name": "HIVE", 23 | "host_count": 1 24 | }, 25 | { 26 | "name": "DN", 27 | "host_count": 1000, 28 | "host_predicate": "Hosts/os_type=centos7" 29 | } 30 | ] 31 | } 32 | -------------------------------------------------------------------------------- /ambari_blueprints/hostmappings-hdfs-ha.json: -------------------------------------------------------------------------------- 1 | { 2 | "blueprint": "cluster-ha-named", 3 | "default_password": "test", 4 | "host_groups": [ 5 | { 6 | "name": "NN1", 7 | "hosts": [ 8 | { 9 | "fqdn": "host1.internal" 10 | } 11 | ] 12 | }, 13 | { 14 | "name": "NN2", 15 | "hosts": [ 16 | { 17 | "fqdn": "host2.internal" 18 | } 19 | ] 20 | }, 21 | { 22 | "name": "QJM3", 23 | "hosts": [ 24 | { 25 | "fqdn": "host3.internal" 26 | } 27 | ] 28 | }, 29 | { 30 | "name": "HIVE", 31 | "hosts": [ 32 | { 33 | "fqdn": "host4.internal" 34 | } 35 | ] 36 | }, 37 | { 38 | "name": "DN", 39 | "host_count": 1000, 40 | "host_predicate": "Hosts/os_type=centos7" 41 | } 42 | ] 43 | } 44 | -------------------------------------------------------------------------------- /ambari_blueprints/hostmappings-hdfs-yarn-ha-random-hosts.json: -------------------------------------------------------------------------------- 1 | { 2 | "blueprint": "cluster-hdfs-yarn-ha", 3 | "default_password": "test", 4 | "host_groups": [ 5 | { 6 | "name": "AMBARI", 7 | "hosts": [ 8 | { 9 | "fqdn": "host1.internal" 10 | } 11 | ] 12 | }, 13 | { 14 | "name": "MASTER1", 15 | "host_count": 1 16 | }, 17 | { 18 | "name": "MASTER2", 19 | "host_count": 1 20 | }, 21 | { 22 | "name": "MASTER3", 23 | "host_count": 1 24 | }, 25 | { 26 | "name": "METRICS", 27 | "host_count": 1 28 | }, 29 | { 30 | "name": "DN", 31 | "host_count": 1000, 32 | "host_predicate": "Hosts/os_type=centos7" 33 | } 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /ambari_blueprints/hostmappings-hdfs-yarn-ha.json: -------------------------------------------------------------------------------- 1 | { 2 | "blueprint": "cluster-hdfs-yarn-ha", 3 | "default_password": "test", 4 | "host_groups": [ 5 | { 6 | "name": "MASTER1", 7 | "hosts": [ 8 | { 9 | "fqdn": "host1.internal" 10 | } 11 | ] 12 | }, 13 | { 14 | "name": "MASTER2", 15 | "hosts": [ 16 | { 17 | "fqdn": "host2.internal" 18 | } 19 | ] 20 | }, 21 | { 22 | "name": "MASTER3", 23 | "hosts": [ 24 | { 25 | "fqdn": "host3.internal" 26 | } 27 | ] 28 | }, 29 | { 30 | "name": "METRICS", 31 | "hosts": [ 32 | { 33 | "fqdn": "host4.internal" 34 | } 35 | ] 36 | }, 37 | { 38 | "name": "AMBARI", 39 | "hosts": [ 40 | { 41 | "fqdn": "host5.internal" 42 | } 43 | ] 44 | }, 45 | { 46 | "name": "DN", 47 | "host_count": 1000, 48 | "host_predicate": "Hosts/os_type=centos7" 49 | } 50 | ] 51 | } 52 | -------------------------------------------------------------------------------- /ambari_blueprints/hostmappings-hdfs-yarn-hbase-ha-random-hosts.json: -------------------------------------------------------------------------------- 1 | { 2 | "blueprint": "cluster-hdfs-yarn-ha", 3 | "default_password": "test", 4 | "host_groups": [ 5 | { 6 | "name": "AMBARI", 7 | "hosts": [ 8 | { 9 | "fqdn": "host1.internal" 10 | } 11 | ] 12 | }, 13 | { 14 | "name": "MASTER1", 15 | "host_count": 1 16 | }, 17 | { 18 | "name": "MASTER2", 19 | "host_count": 1 20 | }, 21 | { 22 | "name": "MASTER3", 23 | "host_count": 1 24 | }, 25 | { 26 | "name": "METRICS", 27 | "host_count": 1 28 | }, 29 | { 30 | "name": "DN", 31 | "host_count": 1000, 32 | "host_predicate": "Hosts/os_type=centos7" 33 | } 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /ambari_blueprints/hostmappings-hdfs-yarn-hbase-ha.json: -------------------------------------------------------------------------------- 1 | hostmappings-hdfs-yarn-ha.json -------------------------------------------------------------------------------- /ambari_blueprints/hostmappings-named-random-hosts.json: -------------------------------------------------------------------------------- 1 | { 2 | "blueprint": "cluster-basic-named", 3 | "default_password": "test", 4 | "host_groups": [ 5 | { 6 | "name": "AMBARI", 7 | "hosts": [ 8 | { 9 | "fqdn": "host1.internal" 10 | } 11 | ] 12 | }, 13 | { 14 | "name": "NN", 15 | "host_count": 1 16 | }, 17 | { 18 | "name": "SNN", 19 | "host_count": 1 20 | }, 21 | { 22 | "name": "HIVE", 23 | "host_count": 1 24 | }, 25 | { 26 | "name": "METRICS", 27 | "host_count": 1 28 | }, 29 | { 30 | "name": "DN", 31 | "host_count": 1000, 32 | "host_predicate": "Hosts/os_type=centos7" 33 | } 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /ambari_blueprints/hostmappings-named.json: -------------------------------------------------------------------------------- 1 | { 2 | "blueprint": "cluster-basic-named", 3 | "default_password": "test", 4 | "host_groups": [ 5 | { 6 | "name": "NN", 7 | "hosts": [ 8 | { 9 | "fqdn": "host1.internal" 10 | } 11 | ] 12 | }, 13 | { 14 | "name": "SNN", 15 | "hosts": [ 16 | { 17 | "fqdn": "host2.internal" 18 | } 19 | ] 20 | }, 21 | { 22 | "name": "HIVE", 23 | "hosts": [ 24 | { 25 | "fqdn": "host3.internal" 26 | } 27 | ] 28 | }, 29 | { 30 | "name": "METRICS", 31 | "hosts": [ 32 | { 33 | "fqdn": "host4.internal" 34 | } 35 | ] 36 | }, 37 | { 38 | "name": "DN", 39 | "host_count": 1000, 40 | "host_predicate": "Hosts/os_type=centos7" 41 | }, 42 | { 43 | "name": "AMBARI", 44 | "hosts": [ 45 | { 46 | "fqdn": "host6.internal" 47 | } 48 | ] 49 | } 50 | ] 51 | } 52 | -------------------------------------------------------------------------------- /ambari_blueprints/hostmappings-random-hosts.json: -------------------------------------------------------------------------------- 1 | { 2 | "blueprint": "cluster-basic", 3 | "default_password": "test", 4 | "host_groups": [ 5 | { 6 | "name": "host_group_6", 7 | "hosts": [ 8 | { 9 | "fqdn": "host6.internal" 10 | } 11 | ] 12 | }, 13 | { 14 | "name": "host_group_1", 15 | "host_count": 1 16 | }, 17 | { 18 | "name": "host_group_2", 19 | "host_count": 1 20 | }, 21 | { 22 | "name": "host_group_3", 23 | "host_count": 1 24 | }, 25 | { 26 | "name": "host_group_4", 27 | "host_count": 1 28 | }, 29 | { 30 | "name": "host_group_5", 31 | "host_count": 1000, 32 | "host_predicate": "Hosts/os_type=centos7" 33 | } 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /ambari_blueprints/hostmappings.json: -------------------------------------------------------------------------------- 1 | { 2 | "blueprint": "cluster-basic", 3 | "default_password": "test", 4 | "host_groups": [ 5 | { 6 | "name": "host_group_1", 7 | "hosts": [ 8 | { 9 | "fqdn": "host1.internal" 10 | } 11 | ] 12 | }, 13 | { 14 | "name": "host_group_2", 15 | "hosts": [ 16 | { 17 | "fqdn": "host2.internal" 18 | } 19 | ] 20 | }, 21 | { 22 | "name": "host_group_3", 23 | "hosts": [ 24 | { 25 | "fqdn": "host3.internal" 26 | } 27 | ] 28 | }, 29 | { 30 | "name": "host_group_4", 31 | "hosts": [ 32 | { 33 | "fqdn": "host4.internal" 34 | } 35 | ] 36 | }, 37 | { 38 | "name": "host_group_5", 39 | "host_count": 1000, 40 | "host_predicate": "Hosts/os_type=centos7" 41 | }, 42 | { 43 | "name": "host_group_6", 44 | "hosts": [ 45 | { 46 | "fqdn": "host6.internal" 47 | } 48 | ] 49 | } 50 | ] 51 | } 52 | -------------------------------------------------------------------------------- /anonymize_ignore.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2013-07-18 22:00:56 +0100 (Thu, 18 Jul 2013) 4 | # 5 | # https://github.com/HariSekhon/DevOps-Python-tools 6 | # 7 | # License: see accompanying LICENSE file 8 | # 9 | 10 | # anonymize_ignore.conf - used by anonymize.py 11 | # 12 | # List of phrases to ignore anonymization, one PCRE format Regex per line 13 | # 14 | # Regex are case insensitive 15 | # 16 | # blank lines and lines prefixed with a hash sign (#) are ignored 17 | 18 | # Implemented in the form of negative lookaheads as variable length lookbehind isn't implemented in Python regex engine 19 | # 20 | # XXX: Be careful not to add sloppy regex like .* or .+ in here as it will prevent anonymization. Use this file sparingly, it's better to code generic tested rules in to the relative subroutines where possible 21 | # 22 | # XXX: Also be careful to not put capturing brackets in here as it may short circuit replacement operations 23 | # 24 | # XXX: if adding anything to this file, it's recommended to run tests/test_anonymize.sh to see if it breaks existing stuff 25 | # 26 | # XXX: this currently only applies to --hostname/--domain/--fqdn as they often match non-host related things like class names in debug messages that you want to preserve for pasting in Apache Jira tickets etc 27 | 28 | #Script=/var/lib/ambari-server/resources/scripts/ 29 | \S*exception 30 | #\bmain\.(?:py|java) 31 | #\b[A-Z][a-z]+(?:[A-Z][a-z]+)+\.(?:py|java):\d+ 32 | \b[A-Za-z0-9_-]+\.(?:py|java):\d+ 33 | # ignore versions like /usr/hdp/2.3.0 - now handled 34 | /\w+/\w+/\d+\.\d+\.\d+\.\d+ 35 | -------------------------------------------------------------------------------- /bitbucket-pipelines.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2020-02-24 17:08:57 +0000 (Mon, 24 Feb 2020) 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | # ============================================================================ # 17 | # B i t b u c k e t C I / C D P i p e l i n e 18 | # ============================================================================ # 19 | 20 | # Reference: 21 | # 22 | # https://support.atlassian.com/bitbucket-cloud/docs/configure-bitbucket-pipelinesyml/ 23 | 24 | # Languages: 25 | # 26 | # https://confluence.atlassian.com/x/5Q4SMw 27 | 28 | # You can specify a custom docker image from Docker Hub as your build environment. 29 | image: atlassian/default-image:2 30 | 31 | pipelines: 32 | default: 33 | - step: 34 | script: 35 | - setup/ci_bootstrap.sh 36 | - make init 37 | - make ci 38 | - make test 39 | -------------------------------------------------------------------------------- /boot: -------------------------------------------------------------------------------- 1 | setup/bootstrap.sh -------------------------------------------------------------------------------- /buddy.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2020-03-16 14:02:53 +0000 (Mon, 16 Mar 2020) 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | # ============================================================================ # 17 | # B u d d y C I 18 | # ============================================================================ # 19 | 20 | # https://buddy.works/docs/yaml/yaml-schema 21 | 22 | --- 23 | - pipeline: "Build" 24 | trigger_mode: "ON_EVERY_PUSH" 25 | ref_name: "master" 26 | ref_type: "BRANCH" 27 | target_site_url: "https://github.com/HariSekhon/DevOps-Python-tools" 28 | trigger_condition: "ALWAYS" 29 | actions: 30 | - action: "Execute: make ci test" 31 | type: "BUILD" 32 | working_directory: "/buddy/DevOps-Python-tools" 33 | docker_image_name: "library/ubuntu" 34 | docker_image_tag: "18.04" 35 | #setup_commands: 36 | # this step gets cached, which results in 37 | # E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing? 38 | # - apt update 39 | # - apt install -qy git make 40 | execute_commands: 41 | - setup/ci_bootstrap.sh 42 | - make init 43 | - make ci 44 | - make test 45 | volume_mappings: 46 | - "/:/buddy/DevOps-Python-tools" 47 | shell: "BASH" 48 | trigger_condition: "ALWAYS" 49 | -------------------------------------------------------------------------------- /cicd/.concourse.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2020-03-21 11:06:48 +0000 (Sat, 21 Mar 2020) 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | # ============================================================================ # 17 | # C o n c o u r s e C I 18 | # ============================================================================ # 19 | 20 | # https://concourse-ci.org/golang-library-example.html 21 | 22 | # https://resource-types.concourse-ci.org/ 23 | # https://concourse-ci.org/resource-types.html 24 | resources: 25 | - name: github 26 | icon: github-circle 27 | type: git 28 | source: 29 | uri: https://github.com/HariSekhon/DevOps-Python-tools 30 | branch: master 31 | #- name: daily 32 | # type: time 33 | # source: 34 | # interval: 1d 35 | 36 | # https://concourse-ci.org/jobs.html 37 | jobs: 38 | - name: build 39 | public: false 40 | plan: 41 | - get: github # from resource above 42 | trigger: true 43 | #version: every # build every git commit, default: latest 44 | - task: build 45 | config: 46 | platform: linux 47 | image_resource: 48 | type: docker-image 49 | source: 50 | repository: ubuntu 51 | tag: latest 52 | inputs: 53 | - name: github 54 | path: code 55 | params: 56 | CONCOURSE: 1 57 | run: 58 | path: /bin/bash 59 | args: 60 | - -c 61 | - | 62 | cd code && 63 | setup/ci_bootstrap.sh && 64 | make init && 65 | make ci test 66 | -------------------------------------------------------------------------------- /cicd/cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2020-12-19 16:27:26 +0000 (Sat, 19 Dec 2020) 4 | # 5 | # https://github.com/HariSekhon/DevOps-Python-tools 6 | # 7 | # License: see accompanying Hari Sekhon LICENSE file 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | # ============================================================================ # 15 | # G C P C l o u d B u i l d 16 | # ============================================================================ # 17 | 18 | # References: 19 | # 20 | # https://cloud.google.com/cloud-build/docs/build-config 21 | # 22 | # https://cloud.google.com/cloud-build/docs/build-debug-locally 23 | 24 | 25 | # gcloud builds submit --config cloudbuild.yaml . 26 | # 27 | # cloud-build-local --config cloudbuild.yml --dryrun=false . 28 | 29 | # tars $PWD to bucket called ${PROJECT_ID}_cloudbuild 30 | 31 | timeout: 3660s 32 | 33 | steps: 34 | - name: ubuntu:18.04 35 | entrypoint: bash 36 | args: 37 | - '-c' 38 | - | 39 | setup/ci_bootstrap.sh && 40 | make build test 41 | timeout: 3600s 42 | -------------------------------------------------------------------------------- /cloudformation/README.md: -------------------------------------------------------------------------------- 1 | Hari Sekhon CloudFormation Templates 2 | ==================================== 3 | 4 | These are for testing only and should not be blindly used for production. 5 | 6 | CloudFormation templates should be tailored by an appropriately qualified and experienced engineer for production use (it would be expensive to run production specs on AWS hence I tone down the specs a lot). 7 | -------------------------------------------------------------------------------- /codefresh.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2020-02-24 17:43:07 +0000 (Mon, 24 Feb 2020) 4 | # 5 | # vim:ts=2:sts=2:sw=2:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | # ============================================================================ # 17 | # C o d e f r e s h C I 18 | # ============================================================================ # 19 | 20 | # https://codefresh.io/docs/docs/codefresh-yaml/ 21 | 22 | version: "1.0" 23 | stages: 24 | - "checkout" 25 | - "build" 26 | steps: 27 | checkout: 28 | type: "git-clone" 29 | description: "Cloning main repository..." 30 | repo: '${{CF_REPO_OWNER}}/${{CF_REPO_NAME}}' 31 | revision: "${{CF_REVISION}}" 32 | stage: "checkout" 33 | build: 34 | title: Running docker image 35 | type: freestyle 36 | working_directory: '${{CF_REPO_NAME}}' 37 | arguments: 38 | image: 'ubuntu:18.04' 39 | commands: 40 | - setup/ci_bootstrap.sh 41 | - make init 42 | - make ci 43 | - make test 44 | -------------------------------------------------------------------------------- /crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2020-05-29 12:35:16 +0100 (Fri, 29 May 2020) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 19 | 20 | # shellcheck disable=SC1090 21 | . "$srcdir/lib.sh" 22 | 23 | # statements should be named in format: Barclaycard_Statement_YYYY-MM-DD.csv 24 | export STATEMENT_GLOB="Barclaycard_Statement_[[:digit:]][[:digit:]][[:digit:]][[:digit:]]-[[:digit:]][[:digit:]]-[[:digit:]][[:digit:]].csv" 25 | 26 | generate_crunch_statements --credit-card --reverse-order 27 | -------------------------------------------------------------------------------- /crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2020-05-29 12:35:16 +0100 (Fri, 29 May 2020) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 19 | 20 | # shellcheck disable=SC1090 21 | . "$srcdir/lib.sh" 22 | 23 | # statements should be named in format: Barclays_Statement_YYYY-MM-DD.csv 24 | export STATEMENT_GLOB="Barclays_Statement_[[:digit:]][[:digit:]][[:digit:]][[:digit:]]-[[:digit:]][[:digit:]]-[[:digit:]][[:digit:]].csv" 25 | 26 | # Barclays CSV statements often have whitespace starting fields instead of blank or 'null' 27 | # unfortunately this resets all the original CSV timestamps each run so it's best to do only where needed 28 | # UPDATE: no longer necessary, converter just ignores these blank fields now in the validation 29 | #for statement in $STATEMENT_GLOB; do 30 | #perl -pi -e 's/^\s+,/,/' "$statement" 31 | #done 32 | 33 | generate_crunch_statements --reverse-order 34 | -------------------------------------------------------------------------------- /docker_pull_all_images.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2017-08-29 14:57:23 +0200 (Tue, 29 Aug 2017) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | 19 | srcdir="$(dirname "$0")" 20 | 21 | for docker_image in $("$srcdir/dockerhub_search.py" -q harisekhon); do 22 | echo "docker pull $docker_image" 23 | docker pull "$docker_image" 24 | echo 25 | done 26 | -------------------------------------------------------------------------------- /docker_pull_all_images_all_tags.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2017-08-29 14:57:23 +0200 (Tue, 29 Aug 2017) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | 19 | srcdir="$(dirname "$0")" 20 | 21 | for docker_image in $("$srcdir/dockerhub_search.py" -q harisekhon | grep -v 'presto.*dev'); do 22 | echo "pulling all tags for docker image $docker_image:" 23 | for tag in $("$srcdir/dockerhub_show_tags.py" -q "$docker_image"); do 24 | echo 25 | echo "docker pull $docker_image:$tag" 26 | docker pull "$docker_image:$tag" 27 | done 28 | echo 29 | echo 30 | done 31 | -------------------------------------------------------------------------------- /docker_pull_all_tags.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2017-08-29 14:57:23 +0200 (Tue, 29 Aug 2017) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | 19 | srcdir="$(dirname "$0")" 20 | 21 | docker_image="${1:-}" 22 | 23 | if [ -z "$docker_image" ]; then 24 | echo "usage: $0 " 25 | exit 1 26 | fi 27 | 28 | for tag in $("$srcdir/dockerhub_show_tags.py" -q "$docker_image"); do 29 | echo "docker pull $docker_image:$tag" 30 | docker pull "$docker_image:$tag" 31 | echo 32 | done 33 | -------------------------------------------------------------------------------- /gcp_cloud_function_ifconfig/.gcloudignore: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2020-10-16 11:44:51 +0100 (Fri, 16 Oct 2020) 4 | # 5 | # vim:ts=4:sts=4:sw=4:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | # see also: massive generic .gcloudignore at https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.gcloudignore 17 | 18 | .git 19 | .gcloudignore 20 | deploy.sh 21 | test/ 22 | tests/ 23 | -------------------------------------------------------------------------------- /gcp_cloud_function_ifconfig/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2021-01-18 18:15:39 +0000 (Mon, 18 Jan 2021) 4 | # 5 | # vim:ts=4:sts=4:sw=4:noet 6 | # 7 | # https://github.com/HariSekhon/Kubernetes-templates 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | SHELL = /usr/bin/env bash 15 | 16 | .PHONY: default 17 | default: deploy 18 | @: 19 | 20 | .PHONY: deploy 21 | deploy: 22 | @./deploy.sh 23 | -------------------------------------------------------------------------------- /gcp_cloud_function_ifconfig/README.md: -------------------------------------------------------------------------------- 1 | Google Cloud Function - ifconfig 2 | ===================== 3 | 4 | Queries http://ifconfig.co from GCF to check the routing and external IP being used eg. for comparison with Cloudflare / Firewall rules 5 | 6 | - `main.py` - the code 7 | - `requirements.txt` - the pip modules to bootstrap 8 | - `deploy.sh` - upload the code and deps 9 | 10 | Response is HTTP status code and message, then the raw JSON results 11 | 12 | ``` 13 | 200 OK 14 | 15 | { 16 | "ip": "1.2.3.4", 17 | "ip_decimal": 1234567890, 18 | "country": "United States", 19 | "country_iso": "US", 20 | "country_eu": false, 21 | "latitude": 37.751, 22 | "longitude": -97.822, 23 | "time_zone": "America/Chicago", 24 | "asn": "AS15169", 25 | "asn_org": "GOOGLE", 26 | "hostname": "ipv6.gae.googleusercontent.com", 27 | "user_agent": { 28 | "product": "python-requests", 29 | "version": "2.24.0", 30 | "raw_value": "python-requests/2.24.0" 31 | } 32 | } 33 | ``` 34 | 35 | For IPv6 the format will be more like: 36 | ``` 37 | "ip": "1234:5678:9012:34::a", 38 | "ip_decimal": 12345678901234567890123456789012345678, 39 | ``` 40 | 41 | Upload the function to GCF in the current GCP project - this script will call `gcloud functions deploy` with the required switches: 42 | 43 | ``` 44 | ./deploy.sh 45 | ``` 46 | -------------------------------------------------------------------------------- /gcp_cloud_function_ifconfig/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2021-05-24 16:34:19 +0100 (Mon, 24 May 2021) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 19 | 20 | cd "$srcdir" 21 | 22 | name="ifconfig" 23 | 24 | # Cloud Functions not available in all regions yet: 25 | # 26 | # https://cloud.google.com/functions/docs/locations 27 | # 28 | # gcloud functions deploy doesn't seem to infer CLOUDSDK_COMPUTE_REGION from environment 29 | region="$(gcloud config list --format="value(compute.region)" 2>&1 || :)" 30 | region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" 31 | 32 | # https://console.cloud.google.com/marketplace/product/google/vpcaccess.googleapis.com 33 | # for serverless VPC access to resources using their Private IPs 34 | # since we're only accessing the SQL Admin API we don't need this 35 | #vpc_connector="my-vpc-connector" 36 | 37 | opts=() 38 | if [ -n "${vpc_connector:-}" ]; then 39 | # routes all traffic through VPC connector to re-use the VPC's Cloud NAT IP eg. for permitting in firewall rules 40 | opts+=(--vpc-connector "$vpc_connector" --egress-settings all) 41 | fi 42 | 43 | set -x 44 | gcloud functions deploy "$name" --trigger-http \ 45 | --security-level=secure-always \ 46 | --runtime python39 \ 47 | --entry-point main \ 48 | --memory 128MB \ 49 | --region "$region" \ 50 | --timeout 60 \ 51 | "${opts[@]}" \ 52 | --quiet # don't prompt to --allow-unauthenticated 53 | #--max-instances 1 \ 54 | -------------------------------------------------------------------------------- /gcp_cloud_function_ifconfig/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | # vim:ts=4:sts=4:sw=4:et 4 | # 5 | # Author: Hari Sekhon 6 | # Date: 2021-05-24 16:03:30 +0100 (Mon, 24 May 2021) 7 | # 8 | # https://github.com/HariSekhon/DevOps-Python-tools 9 | # 10 | # License: see accompanying Hari Sekhon LICENSE file 11 | # 12 | # If you're using my code you're welcome to connect with me on LinkedIn 13 | # and optionally send me feedback to help steer this or other code I publish 14 | # 15 | # https://www.linkedin.com/in/HariSekhon 16 | # 17 | 18 | """ 19 | 20 | GCP Cloud Function to query ifconfig.co to show our IP information for debugging VPC Connector access 21 | routing via specified VPC Network to using default NAT Gateway 22 | 23 | Example usage: check GCF source IP to compare if it's permitted through Cloudflare / Firewall rules 24 | 25 | The HTTP request is irrelevant, just pass an empty JSON document '{}', although we could pass the website to query 26 | in a field in which case this would just act as a proxy. 27 | 28 | See Also: gcp_cloud_function_proxy/main.py 29 | 30 | 31 | Tested on GCP Cloud Functions with Python 3.9 32 | 33 | """ 34 | 35 | # https://cloud.google.com/functions/docs/writing/http#writing_http_content-python 36 | 37 | # https://cloud.google.com/functions/docs/writing/specifying-dependencies-python 38 | 39 | import requests 40 | 41 | def main(_): 42 | """Responds to any HTTP request. 43 | Args: 44 | request (flask.Request): HTTP request object. 45 | Returns: 46 | The response text or any set of values that can be turned into a 47 | Response object using 48 | `make_response `. 49 | """ 50 | req = requests.get('http://ifconfig.co/json') # show our IP information for debugging VPC connector routing 51 | status_code = req.status_code 52 | status_message = req.reason 53 | content = req.text 54 | return "{} {}\n\n{}".format(status_code, status_message, content) 55 | -------------------------------------------------------------------------------- /gcp_cloud_function_ifconfig/requirements.txt: -------------------------------------------------------------------------------- 1 | # https://cloud.google.com/functions/docs/writing/specifying-dependencies-python 2 | requests==2.24.0 3 | -------------------------------------------------------------------------------- /gcp_cloud_function_proxy/.gcloudignore: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2020-10-16 11:44:51 +0100 (Fri, 16 Oct 2020) 4 | # 5 | # vim:ts=4:sts=4:sw=4:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | # see also: massive generic .gcloudignore at https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.gcloudignore 17 | 18 | .git 19 | .gcloudignore 20 | deploy.sh 21 | test/ 22 | tests/ 23 | -------------------------------------------------------------------------------- /gcp_cloud_function_proxy/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2021-01-18 18:15:39 +0000 (Mon, 18 Jan 2021) 4 | # 5 | # vim:ts=4:sts=4:sw=4:noet 6 | # 7 | # https://github.com/HariSekhon/Kubernetes-templates 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | SHELL = /usr/bin/env bash 15 | 16 | .PHONY: default 17 | default: deploy 18 | @: 19 | 20 | .PHONY: deploy 21 | deploy: 22 | @./deploy.sh 23 | -------------------------------------------------------------------------------- /gcp_cloud_function_proxy/README.md: -------------------------------------------------------------------------------- 1 | Google Cloud Function - Proxy 2 | ===================== 3 | 4 | Queries a given URL from GCF to check connectivity eg. for testing with Cloudflare / Firewall rules 5 | 6 | Query content: 7 | ``` 8 | {"url": "http://ifconfig.co/json"} 9 | ``` 10 | 11 | Response is HTTP status code and message, blank line and then the content: 12 | 13 | ``` 14 | 200 OK 15 | 16 | 17 | ``` 18 | 19 | - `main.py` - the code 20 | - `requirements.txt` - the pip modules to bootstrap 21 | - `deploy.sh` - upload the code and deps 22 | 23 | Upload the function to GCF in the current GCP project - this script will call `gcloud functions deploy` with the required switches: 24 | 25 | ``` 26 | ./deploy.sh 27 | ``` 28 | -------------------------------------------------------------------------------- /gcp_cloud_function_proxy/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2021-05-24 16:34:19 +0100 (Mon, 24 May 2021) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 19 | 20 | cd "$srcdir" 21 | 22 | name="proxy" 23 | 24 | # Cloud Functions not available in all regions yet: 25 | # 26 | # https://cloud.google.com/functions/docs/locations 27 | # 28 | # gcloud functions deploy doesn't seem to infer CLOUDSDK_COMPUTE_REGION from environment 29 | region="$(gcloud config list --format="value(compute.region)" 2>&1 || :)" 30 | region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" 31 | 32 | # https://console.cloud.google.com/marketplace/product/google/vpcaccess.googleapis.com 33 | # for serverless VPC access to resources using their Private IPs 34 | # since we're only accessing the SQL Admin API we don't need this 35 | #vpc_connector="my-vpc-connector" 36 | 37 | opts=() 38 | if [ -n "${vpc_connector:-}" ]; then 39 | # routes all traffic through VPC connector to re-use the VPC's Cloud NAT IP eg. for permitting in firewall rules 40 | opts+=(--vpc-connector "$vpc_connector" --egress-settings all) 41 | fi 42 | 43 | set -x 44 | gcloud functions deploy "$name" --trigger-http \ 45 | --security-level=secure-always \ 46 | --runtime python39 \ 47 | --entry-point main \ 48 | --memory 128MB \ 49 | --region "$region" \ 50 | --timeout 60 \ 51 | "${opts[@]}" \ 52 | --quiet # don't prompt to --allow-unauthenticated 53 | #--max-instances 1 54 | -------------------------------------------------------------------------------- /gcp_cloud_function_proxy/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | # vim:ts=4:sts=4:sw=4:et 4 | # 5 | # Author: Hari Sekhon 6 | # Date: 2021-05-24 16:03:30 +0100 (Mon, 24 May 2021) 7 | # 8 | # https://github.com/HariSekhon/DevOps-Python-tools 9 | # 10 | # License: see accompanying Hari Sekhon LICENSE file 11 | # 12 | # If you're using my code you're welcome to connect with me on LinkedIn 13 | # and optionally send me feedback to help steer this or other code I publish 14 | # 15 | # https://www.linkedin.com/in/HariSekhon 16 | # 17 | 18 | """ 19 | 20 | GCP Cloud Function to query ifconfig.co to show our IP information for debugging VPC Connector access 21 | routing via specified VPC Network to using default NAT Gateway 22 | 23 | Example usage: 24 | 25 | Check GCF source IP to compare if it's permitted through Cloudflare / Firewall rules 26 | 27 | Test request examples: 28 | 29 | { "url": "http://ifconfig.co/json" } 30 | 31 | defaults to http:// if not specified: 32 | 33 | { "url": "ifconfig.co/json" } 34 | 35 | 36 | Tested on GCP Cloud Functions with Python 3.9 37 | 38 | """ 39 | 40 | # https://cloud.google.com/functions/docs/writing/http#writing_http_content-python 41 | 42 | # https://cloud.google.com/functions/docs/writing/specifying-dependencies-python 43 | 44 | import json 45 | import requests 46 | 47 | def main(request): 48 | """Responds to any HTTP request. 49 | Args: 50 | request (flask.Request): HTTP request object. 51 | Returns: 52 | The response text or any set of values that can be turned into a 53 | Response object using 54 | `make_response `. 55 | """ 56 | data = json.loads(request.data) 57 | url = data['url'] 58 | if '://' not in url: 59 | url = 'http://' + url 60 | req = requests.get(url) 61 | status_code = req.status_code 62 | status_message = req.reason 63 | content = req.text 64 | return "{} {}\n\n{}".format(status_code, status_message, content) 65 | -------------------------------------------------------------------------------- /gcp_cloud_function_proxy/requirements.txt: -------------------------------------------------------------------------------- 1 | # https://cloud.google.com/functions/docs/writing/specifying-dependencies-python 2 | requests==2.24.0 3 | -------------------------------------------------------------------------------- /gcp_cloud_function_sql_export/.gcloudignore: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2020-10-16 11:44:51 +0100 (Fri, 16 Oct 2020) 4 | # 5 | # vim:ts=4:sts=4:sw=4:et 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | # see also: massive generic .gcloudignore at https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.gcloudignore 17 | 18 | .git 19 | .gcloudignore 20 | deploy.sh 21 | test/ 22 | tests/ 23 | -------------------------------------------------------------------------------- /gcp_cloud_function_sql_export/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Author: Hari Sekhon 3 | # Date: 2021-01-18 18:15:39 +0000 (Mon, 18 Jan 2021) 4 | # 5 | # vim:ts=4:sts=4:sw=4:noet 6 | # 7 | # https://github.com/HariSekhon/Kubernetes-templates 8 | # 9 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 10 | # 11 | # https://www.linkedin.com/in/HariSekhon 12 | # 13 | 14 | SHELL = /usr/bin/env bash 15 | 16 | .PHONY: default 17 | default: deploy 18 | @: 19 | 20 | .PHONY: deploy 21 | deploy: 22 | @./deploy.sh 23 | -------------------------------------------------------------------------------- /gcp_cloud_function_sql_export/requirements.txt: -------------------------------------------------------------------------------- 1 | # https://cloud.google.com/functions/docs/writing/specifying-dependencies-python 2 | google-api-python-client==1.12.5 3 | oauth2client==4.1.3 4 | -------------------------------------------------------------------------------- /hbase_regions_by_memstore_size.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2016-10-06 10:42:59 +0100 (Thu, 06 Oct 2016) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn 12 | # and optionally send me feedback to help steer this or other code I publish 13 | # 14 | # https://www.linkedin.com/in/HariSekhon 15 | # 16 | 17 | """ 18 | 19 | Tool to find HBase Regions memstores sizes across any given RegionServers using JMX API stats 20 | 21 | Argument list should be one or more RegionServers to dump the JMX stats from 22 | 23 | See also hbase_regions_by_size.py 24 | hbase_regions_least_used.py 25 | 26 | Tested on Apache HBase 1.0.3, 1.1.6, 1.2.1, 1.2.2, 1.3.1 27 | 28 | """ 29 | 30 | from __future__ import absolute_import 31 | from __future__ import division 32 | from __future__ import print_function 33 | from __future__ import unicode_literals 34 | 35 | import os 36 | import sys 37 | import traceback 38 | libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) 39 | sys.path.append(libdir) 40 | try: 41 | # pylint: disable=wrong-import-position 42 | from hbase_regions_by_size import HBaseRegionsBySize 43 | except ImportError as _: 44 | print(traceback.format_exc(), end='') 45 | sys.exit(4) 46 | 47 | __author__ = 'Hari Sekhon' 48 | __version__ = '0.6.2' 49 | 50 | 51 | class HBaseRegionsByMemstoreSize(HBaseRegionsBySize): 52 | 53 | def __init__(self): 54 | # Python 2.x 55 | super(HBaseRegionsByMemstoreSize, self).__init__() 56 | # Python 3.x 57 | # super().__init__() 58 | # could have just made this a switch to hbase_regions_by_size.py but it's easier to see as separate prog 59 | self.metric = 'memStoreSize' 60 | 61 | 62 | if __name__ == '__main__': 63 | HBaseRegionsByMemstoreSize().main() 64 | -------------------------------------------------------------------------------- /hbase_scan_table_column_names.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2018-06-29 19:01:22 +0100 (Fri, 29 Jun 2018) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | 19 | usage(){ 20 | [ -n "${*:-}" ] && echo "$@" 21 | echo 22 | echo "Fast and dirty script to scan an HBase table to determine all columns (column family + qualifiers) 23 | 24 | Beware this is a very expensive O(n) operation and you shouldn't be doing this unless you know what you're doing 25 | 26 | usage: $0 --table 27 | usage: $0 -t 28 | " 29 | exit 1 30 | } 31 | 32 | until [ $# -lt 1 ]; do 33 | case $1 in 34 | -t|--table) 35 | table="$2" 36 | shift || : 37 | ;; 38 | *) usage 39 | ;; 40 | esac 41 | shift 42 | done 43 | 44 | if [ -z "${table:-}" ]; then 45 | usage "ERROR: Table not defined, use --table switch" 46 | fi 47 | 48 | if ! type -P hbase &>/dev/null; then 49 | usage "ERROR: hbase command not found in \$PATH ($PATH)" 50 | fi 51 | 52 | hbase shell <<< "scan '$table'" | 53 | sed 's/.*columns=//;s/,.*$//' | 54 | grep -v -e ^$ -e [[:space:]] | 55 | sort -u 56 | -------------------------------------------------------------------------------- /hbase_table_regionserver_distribution.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2018-07-11 19:36:23 +0100 (Wed, 11 Jul 2018) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | # Useful to see table region skew across regionservers 17 | # 18 | # Consider setting for dedicated HBase clusters such as OpenTSDB: 19 | # 20 | # hbase.master.loadbalance.bytable=true 21 | # 22 | # 23 | # Tested on Hortonworks HDP 2.6 (HBase 1.1.2) and Apache HBase 1.3.1 24 | 25 | set -euo pipefail 26 | [ -n "${DEBUG:-}" ] && set -x 27 | 28 | usage(){ 29 | echo "usage: $0 " 30 | exit 1 31 | } 32 | 33 | port="${REGIONSERVER_PORT:-16030}" 34 | 35 | if [ $# -lt 1 ]; then 36 | usage 37 | fi 38 | 39 | for server in "$@"; do 40 | echo "$server:" 41 | curl -s "$server:$port/jmx" | 42 | grep 'Namespace_.*_metric_getCount"[[:space:]]*:[[:space:]]*' | 43 | sed 's/.*_table_//; s/_region_.*//' | 44 | sort | 45 | uniq -c 46 | echo 47 | done 48 | -------------------------------------------------------------------------------- /hbase_uncompacted_regions.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2018-07-11 19:26:21 +0100 (Wed, 11 Jul 2018) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | # Find regions that require major_compact after region migration to achieve data locality again 17 | # 18 | # Tested on Hortonworks HDP 2.6 (HBase 1.1.2) and Apache HBase 1.3.1 19 | 20 | set -euo pipefail 21 | [ -n "${DEBUG:-}" ] && set -x 22 | 23 | usage(){ 24 | echo "usage: $0 " 25 | exit 1 26 | } 27 | 28 | port="${REGIONSERVER_PORT:-16030}" 29 | 30 | if [ $# -lt 1 ]; then 31 | usage 32 | fi 33 | 34 | for server in "$@"; do 35 | curl -s "$server:$port/jmx" | 36 | grep 'compactionsCompletedCount"[[:space:]]*:[[:space:]]*0,$' | 37 | sed 's/.*_table_//; s/_region_/ /; s/_metric_.*//' | 38 | column -t 39 | done 40 | -------------------------------------------------------------------------------- /jython_autoinstall.exp: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env expect 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: Fri Jun 17 15:12:17 2016 +0100 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set force_conservative 0 ;# set to 1 to force conservative mode even if 17 | ;# script wasn't run conservatively originally 18 | if {$force_conservative} { 19 | set send_slow {1 .1} 20 | proc send {ignore arg} { 21 | sleep .1 22 | exp_send -s -- $arg 23 | } 24 | } 25 | 26 | set timeout -1 27 | spawn java -jar jython-installer.jar 28 | match_max 100000 29 | expect -exact "Please select your language \[E/g\] >>> " 30 | send -- "\r" 31 | expect -exact "Do you want to read the license agreement now ? \[y/N\] >>> " 32 | send -- "\r" 33 | expect -exact "Do you accept the license agreement ? \[Y/n\] >>> " 34 | send -- "\r" 35 | expect -exact "Please select the installation type \[ 1 /2/3/9\] >>> " 36 | send -- "2\r" 37 | expect -exact "Do you want to install additional parts ? \[y/N\] >>> " 38 | send -- "\r" 39 | expect -exact "Please enter the target directory >>> " 40 | send -- "/opt/jython-2.7.0\r" 41 | expect -exact "Unable to find directory /opt/jython-2.7.0, create it ? \[Y/n\] >>> " 42 | send -- "\r" 43 | expect -exact "Please confirm copying of files to directory /opt/jython-2.7.0 \[Y/n\] >>> " 44 | send -- "\r" 45 | expect -exact "Do you want to show the contents of README ? \[y/N\] >>> " 46 | send -- "\r" 47 | expect eof 48 | -------------------------------------------------------------------------------- /jython_install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2016-08-01 10:17:55 +0100 (Mon, 01 Aug 2016) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -eu 17 | [ -n "${DEBUG:-}" ] && set -x 18 | 19 | JYTHON_VERSION="${1:-2.7.0}" 20 | 21 | JYTHON_INSTALL_DIR="${2:-/opt/jython-$JYTHON_VERSION}" 22 | 23 | # not set in busybox 24 | #[ $EUID -eq 0 ] && sudo="" || sudo=sudo 25 | [ "$(whoami)" = "root" ] && sudo="" || sudo=sudo 26 | 27 | # installer will tell us if dir isn't empty 28 | #if ! [ -e "$JYTHON_INSTALL_DIR" ]; then 29 | $sudo mkdir -p "$JYTHON_INSTALL_DIR" 30 | wget -cO jython-installer.jar "http://search.maven.org/remotecontent?filepath=org/python/jython-installer/$JYTHON_VERSION/jython-installer-$JYTHON_VERSION.jar" 31 | #$sudo expect "$srcdir/jython_autoinstall.exp" 32 | # 33 | # 'core' = too minimal to be useful to my real world programs, install results in: 34 | # import socket 35 | # ... 36 | # ImportError: No module named encodings 37 | # 38 | #$sudo java -jar jython-installer.jar --silent --include mod --include ensurepip --directory "$JYTHON_INSTALL_DIR" 39 | $sudo java -jar jython-installer.jar -s -t standard -d "$JYTHON_INSTALL_DIR" 40 | $sudo rm -fr "$JYTHON_INSTALL_DIR"/{Docs,Demo,tests} 41 | $sudo ln -sf "$JYTHON_INSTALL_DIR" /opt/jython 42 | rm -f jython-installer.jar 43 | echo 44 | echo "Jython Install done" 45 | #else 46 | # echo "$JYTHON_INSTALL_DIR already exists - doing nothing" 47 | #fi 48 | if ! [ -e /etc/profile.d/jython.sh ]; then 49 | echo "Adding /etc/profile.d/jython.sh" 50 | # shell execution tracing comes out in the file otherwise 51 | set +x 52 | cat >> /etc/profile.d/jython.sh <", 9 | "description": "Contains all other projects", 10 | "href": "/httpAuth/app/rest/projects/id:_Root", 11 | "webUrl": "http://localhost:8111/project.html?projectId=_Root" 12 | }, 13 | "properties": { 14 | "count": 9, 15 | "property": [ 16 | { 17 | "name": "agentCleanFilesPolicy", 18 | "value": "ALL_UNTRACKED" 19 | }, 20 | { 21 | "name": "agentCleanPolicy", 22 | "value": "ON_BRANCH_CHANGE" 23 | }, 24 | { 25 | "name": "authMethod", 26 | "value": "ANONYMOUS" 27 | }, 28 | { 29 | "name": "branch", 30 | "value": "master" 31 | }, 32 | { 33 | "name": "ignoreKnownHosts", 34 | "value": "true" 35 | }, 36 | { 37 | "name": "submoduleCheckout", 38 | "value": "CHECKOUT" 39 | }, 40 | { 41 | "name": "url", 42 | "value": "https://github.com/HariSekhon/TeamCity-CI" 43 | }, 44 | { 45 | "name": "useAlternates", 46 | "value": "true" 47 | }, 48 | { 49 | "name": "usernameStyle", 50 | "value": "USERID" 51 | } 52 | ] 53 | }, 54 | "vcsRootInstances": { 55 | "href": "/httpAuth/app/rest/vcs-root-instances?locator=vcsRoot:(id:TeamCity)" 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /teamcity/.teamcity.vcs.oauth.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "TeamCity", 3 | "name": "TeamCity", 4 | "vcsName": "jetbrains.git", 5 | "href": "/httpAuth/app/rest/vcs-roots/id:TeamCity", 6 | "project": { 7 | "id": "_Root", 8 | "name": "", 9 | "description": "Contains all other projects", 10 | "href": "/httpAuth/app/rest/projects/id:_Root", 11 | "webUrl": "http://localhost:8111/project.html?projectId=_Root" 12 | }, 13 | "properties": { 14 | "count": 9, 15 | "property": [ 16 | { 17 | "name": "agentCleanFilesPolicy", 18 | "value": "ALL_UNTRACKED" 19 | }, 20 | { 21 | "name": "agentCleanPolicy", 22 | "value": "ON_BRANCH_CHANGE" 23 | }, 24 | { 25 | "name": "authMethod", 26 | "value": "PASSWORD" 27 | }, 28 | { 29 | "name": "branch", 30 | "value": "master" 31 | }, 32 | { 33 | "name": "ignoreKnownHosts", 34 | "value": "true" 35 | }, 36 | { 37 | "name": "submoduleCheckout", 38 | "value": "CHECKOUT" 39 | }, 40 | { 41 | "name": "url", 42 | "value": "https://github.com/HariSekhon/TeamCity-CI" 43 | }, 44 | { 45 | "name": "useAlternates", 46 | "value": "true" 47 | }, 48 | { 49 | "name": "username", 50 | "value": "HariSekhon" 51 | }, 52 | { 53 | "name": "usernameStyle", 54 | "value": "USERID" 55 | } 56 | ] 57 | }, 58 | "vcsRootInstances": { 59 | "href": "/httpAuth/app/rest/vcs-root-instances?locator=vcsRoot:(id:TeamCity)" 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /teamcity/.teamcity.vcs.ssh.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "TeamCity", 3 | "name": "TeamCity", 4 | "vcsName": "jetbrains.git", 5 | "href": "/app/rest/vcs-roots/id:TeamCity", 6 | "project": { 7 | "id": "_Root", 8 | "name": "", 9 | "description": "Contains all other projects", 10 | "href": "/app/rest/projects/id:_Root", 11 | "webUrl": "http://localhost:8111/project.html?projectId=_Root" 12 | }, 13 | "properties": { 14 | "count": 11, 15 | "property": [ 16 | { 17 | "name": "agentCleanFilesPolicy", 18 | "value": "ALL_UNTRACKED" 19 | }, 20 | { 21 | "name": "agentCleanPolicy", 22 | "value": "ON_BRANCH_CHANGE" 23 | }, 24 | { 25 | "name": "authMethod", 26 | "value": "TEAMCITY_SSH_KEY" 27 | }, 28 | { 29 | "name": "branch", 30 | "value": "refs/heads/master" 31 | }, 32 | { 33 | "name": "ignoreKnownHosts", 34 | "value": "true" 35 | }, 36 | { 37 | "name": "submoduleCheckout", 38 | "value": "CHECKOUT" 39 | }, 40 | { 41 | "name": "teamcitySshKey", 42 | "value": "VCS SSH Key" 43 | }, 44 | { 45 | "name": "url", 46 | "value": "github.com:HariSekhon/TeamCity-CI" 47 | }, 48 | { 49 | "name": "useAlternates", 50 | "value": "true" 51 | }, 52 | { 53 | "name": "username", 54 | "value": "git" 55 | }, 56 | { 57 | "name": "usernameStyle", 58 | "value": "USERID" 59 | } 60 | ] 61 | }, 62 | "vcsRootInstances": { 63 | "href": "/app/rest/vcs-root-instances?locator=vcsRoot:(id:TeamCity)" 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /tests/all.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 19 | 20 | # shellcheck disable=SC1090 21 | . "$srcdir/utils.sh" 22 | 23 | # imported by utils.sh above 24 | #. "$srcdir/../bash-tools/lib/utils.sh" 25 | 26 | export PROJECT=pytools 27 | 28 | section "Running PyTools ALL" 29 | 30 | # runs against . by default 31 | cd "$srcdir/.."; 32 | # shellcheck disable=SC1090 33 | # has to be included so that isExcluded function is inherited 34 | . "$srcdir/../bash-tools/checks/check_all.sh" 35 | 36 | #tests/test_yamllint.sh 37 | 38 | # do help afterwards for Spark to be downloaded, and then help will find and use downloaded spark for SPARK_HOME 39 | exit 0 40 | 41 | # pyspark not found 42 | tests/help.sh 43 | 44 | bash-tools/checks/run_tests.sh 45 | -------------------------------------------------------------------------------- /tests/check.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2016-01-16 16:47:43 +0000 (Sat, 16 Jan 2016) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | 19 | check(){ 20 | cmd=$1 21 | msg=$2 22 | echo 23 | hr2 24 | echo "$msg" 25 | hr2 26 | echo 27 | echo "cmd: $cmd" 28 | echo 29 | if eval "$cmd"; then 30 | echo 31 | echo "SUCCESS" 32 | else 33 | echo 34 | echo "FAILED" 35 | exit 1 36 | fi 37 | echo 38 | } 39 | -------------------------------------------------------------------------------- /tests/compile.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -eu 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 19 | 20 | cd "$srcdir/.."; 21 | 22 | # shellcheck disable=SC1090 23 | . "$srcdir/utils.sh" 24 | 25 | hr 26 | echo "Compiling all Python files" 27 | hr 28 | echo 29 | 30 | while read -r filename; do 31 | isExcluded "$filename" && continue 32 | echo "compiling $filename" 33 | python -m py_compile "$filename" 34 | done < <(find . -iname '*.py' -o -iname '*.jy') 35 | echo 36 | echo 37 | -------------------------------------------------------------------------------- /tests/data/add_ou.ldif: -------------------------------------------------------------------------------- 1 | # vim:ts=4:sts=4:sw=4:et 2 | # 3 | # Author: Hari Sekhon 4 | # Date: 2017-08-03 17:44:12 +0200 (Thu, 03 Aug 2017) 5 | # 6 | # https://github.com/HariSekhon/DevOps-Python-tools 7 | # 8 | # License: see accompanying Hari Sekhon LICENSE file 9 | # 10 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 11 | # 12 | # https://www.linkedin.com/in/HariSekhon 13 | # 14 | 15 | dn: OU=Hadoop-Cluster-1,OU=Hadoop,DC=harisekhon,DC=co,DC=uk 16 | ou: Hadoop-Cluster-1 17 | objectClass: top 18 | objectClass: organizationalUnit 19 | -------------------------------------------------------------------------------- /tests/data/embedded_double_quotes.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Hari Sekhon", 3 | "age": 21, 4 | "description": "some \"double quotes\"" 5 | } 6 | -------------------------------------------------------------------------------- /tests/data/header.csv: -------------------------------------------------------------------------------- 1 | Year,Make,Model,Length 2 | 1997,Ford,E350,2.34 3 | 2000,Mercury,Cougar,2.38 4 | -------------------------------------------------------------------------------- /tests/data/header_quoted.csv: -------------------------------------------------------------------------------- 1 | "Year","Make","Model","Length" 2 | "1997","Ford","E350","2.34" 3 | "2000","Mercury","Cougar","2.38" 4 | -------------------------------------------------------------------------------- /tests/data/ldap.ldif: -------------------------------------------------------------------------------- 1 | # extended LDIF 2 | # 3 | # LDAPv3 4 | # base with scope subtree 5 | # filter: (objectclass=*) 6 | # requesting: ALL 7 | # 8 | 9 | # example.org 10 | dn: dc=example,dc=org 11 | objectClass: top 12 | objectClass: dcObject 13 | objectClass: organization 14 | o: Example Inc. 15 | dc: example 16 | 17 | # admin, example.org 18 | dn: cn=admin,dc=example,dc=org 19 | objectClass: simpleSecurityObject 20 | objectClass: organizationalRole 21 | cn: admin 22 | description: LDAP administrator 23 | userPassword:: e1NTSEF9TnpTTnRqbzZUUlgrM2c3Y3c4VHU3eVpCOXhaaDJtN00= 24 | 25 | # Users, example.org 26 | dn: ou=Users,dc=example,dc=org 27 | objectClass: organizationalUnit 28 | objectClass: top 29 | ou: Users 30 | 31 | # Groups, example.org 32 | dn: ou=Groups,dc=example,dc=org 33 | objectClass: organizationalUnit 34 | objectClass: top 35 | ou: Groups 36 | 37 | # Dept1, Groups, example.org 38 | dn: ou=Dept1,ou=Groups,dc=example,dc=org 39 | objectClass: organizationalUnit 40 | objectClass: top 41 | ou: Dept1 42 | 43 | # DeptA, Dept1, Groups, example.org 44 | dn: ou=DeptA,ou=Dept1,ou=Groups,dc=example,dc=org 45 | objectClass: organizationalUnit 46 | objectClass: top 47 | ou: DeptA 48 | 49 | # hari, Users, example.org 50 | dn: cn=hari,ou=Users,dc=example,dc=org 51 | cn: hari 52 | objectClass: organizationalPerson 53 | objectClass: person 54 | objectClass: top 55 | sn: sekhon 56 | memberOf: cn=SuperAdmins,ou=DeptA,ou=Dept1,ou=Groups,dc=example,dc=org 57 | 58 | # SuperAdmins, DeptA, Dept1, Groups, example.org 59 | dn: cn=SuperAdmins,ou=DeptA,ou=Dept1,ou=Groups,dc=example,dc=org 60 | cn: SuperAdmins 61 | member: cn=hari,ou=Users,dc=example,dc=org 62 | objectClass: groupOfNames 63 | objectClass: top 64 | 65 | # search result 66 | #search: 2 67 | #result: 0 Success 68 | 69 | # numResponses: 9 70 | # numEntries: 8 71 | -------------------------------------------------------------------------------- /tests/data/ldap_download.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2019-04-11 18:57:53 +0100 (Thu, 11 Apr 2019) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 19 | 20 | ldapsearch -x -H ldap://localhost -D cn=admin,dc=example,dc=org -w admin -b dc=example,dc=org > "$srcdir/ldap.ldif" 21 | -------------------------------------------------------------------------------- /tests/data/ldap_upload.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2019-04-11 18:57:53 +0100 (Thu, 11 Apr 2019) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 19 | 20 | # -a add 21 | # -c continue on error (if already exists) 22 | 23 | # Insufficient Permissions :-/ 24 | #ldapmodify -a -c -x -H ldap://localhost -D cn=admin,dc=example,dc=org -w admin -f "$srcdir/ldap_memberof_overlay.ldif" 25 | 26 | # add 27 | #ldapmodify -a -c -x -H ldap://localhost -D cn=admin,dc=example,dc=org -w admin -f "$srcdir/ldap.ldif" || : 28 | ldapadd -c -x -H ldap://localhost -D cn=admin,dc=example,dc=org -w admin -f "$srcdir/ldap.ldif" 29 | 30 | # modify 31 | #ldapmodify -c -x -H ldap://localhost -D cn=admin,dc=example,dc=org -w admin -f "$srcdir/ldap.ldif" 32 | -------------------------------------------------------------------------------- /tests/data/multirecord.json: -------------------------------------------------------------------------------- 1 | { "name": "hari" } 2 | { "name": "hari2" } 3 | -------------------------------------------------------------------------------- /tests/data/multirecord_single_double_mixed_quotes.notjson: -------------------------------------------------------------------------------- 1 | { 'name': 'Hari', 'age': 21, 'description': 'single quotes multirecord' } 2 | { "name": "Sekhon", "age": 22, "description": "single quotes multirecord" } 3 | -------------------------------------------------------------------------------- /tests/data/multirecord_single_quotes.notjson: -------------------------------------------------------------------------------- 1 | { 'name': 'Hari', 'age': 21, 'description': 'single quotes multirecord' } 2 | { 'name': 'Sekhon', 'age': 22, 'description': 'single quotes multirecord' } 3 | -------------------------------------------------------------------------------- /tests/data/multirecord_single_quotes_embedded_double_quotes.notjson: -------------------------------------------------------------------------------- 1 | { 'name': 'Hari', 'age': 21, 'description': 'single quotes multirecord \"embedded double quotes\" ' } 2 | { 'name': 'Sekhon', 'age': 22, 'description': 'single quotes multirecord \"embedded double quotes\" ' } 3 | -------------------------------------------------------------------------------- /tests/data/multirecord_single_quotes_embedded_double_quotes_unescaped.notjson: -------------------------------------------------------------------------------- 1 | { 'name': 'Hari', 'age': 21, 'description': 'single quotes multirecord "unescaped embedded double quotes" ' } 2 | { 'name': 'Sekhon', 'age': 22, 'description': 'single quotes multirecord "unescaped embedded double quotes" ' } 3 | -------------------------------------------------------------------------------- /tests/data/multirecord_with_blank_lines.notjson: -------------------------------------------------------------------------------- 1 | { "name": "hari" } 2 | 3 | { "name": "hari2" } 4 | -------------------------------------------------------------------------------- /tests/data/quoted.csv: -------------------------------------------------------------------------------- 1 | "1997","Ford","E350","2.34" 2 | "2000","Mercury","Cougar","2.38" 3 | -------------------------------------------------------------------------------- /tests/data/simple.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Tove 5 | Jani 6 | Reminder 7 | Don't forget me this weekend! 8 | 9 | -------------------------------------------------------------------------------- /tests/data/single_quotes.notjson: -------------------------------------------------------------------------------- 1 | { 2 | 'name': 'Hari Sekhon', 3 | 'age': 21, 4 | 'description': 'single quotes' 5 | } 6 | -------------------------------------------------------------------------------- /tests/data/single_quotes_embedded_double_quotes.notjson: -------------------------------------------------------------------------------- 1 | { 2 | 'name': 'Hari Sekhon', 3 | 'age': 21, 4 | 'description': 'single quotes with embedded \"double quotes\"' 5 | } 6 | -------------------------------------------------------------------------------- /tests/data/single_quotes_embedded_double_quotes_unescaped.notjson: -------------------------------------------------------------------------------- 1 | { 2 | 'name': 'Hari Sekhon', 3 | 'age': 21, 4 | 'description': 'single quotes with embedded "double quotes"' 5 | } 6 | -------------------------------------------------------------------------------- /tests/data/test.csv: -------------------------------------------------------------------------------- 1 | 1997,Ford,E350,2.34 2 | 2000,Mercury,Cougar,2.38 3 | -------------------------------------------------------------------------------- /tests/data/test.ini: -------------------------------------------------------------------------------- 1 | ; 2 | ; comment 3 | ; 4 | 5 | [person] 6 | name = Hari 7 | surname = Sekhon 8 | location=Europe 9 | profession = IT Consultant 10 | -------------------------------------------------------------------------------- /tests/data/test.ini-colons: -------------------------------------------------------------------------------- 1 | ; 2 | ; comment 3 | ; 4 | 5 | [person] 6 | name : Hari 7 | surname : Sekhon 8 | location:Europe 9 | profession : IT Consultant 10 | -------------------------------------------------------------------------------- /tests/data/test.ini-hashes: -------------------------------------------------------------------------------- 1 | # 2 | # comment 3 | # 4 | 5 | [person] 6 | name = Hari 7 | surname = Sekhon 8 | location=Europe 9 | profession = IT Consultant 10 | -------------------------------------------------------------------------------- /tests/data/test.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Hari Sekhon", 3 | "age": 21 4 | } 5 | -------------------------------------------------------------------------------- /tests/data/test.toml: -------------------------------------------------------------------------------- 1 | 2 | title = "TOML Example" 3 | 4 | [owner] 5 | name = "Hari Sekhon" 6 | 7 | [database] 8 | server = "192.168.1.1" 9 | ports = [ 8001, 8001, 8002,] 10 | connection_max = 5000 11 | enabled = true 12 | 13 | [clients] 14 | data = [ [ "gamma", "delta",], [ 1, 2,],] 15 | hosts = [ "alpha", "omega",] 16 | 17 | [servers.alpha] 18 | ip = "10.0.0.1" 19 | dc = "eqdc10" 20 | 21 | [servers.beta] 22 | 23 | ip = "10.0.0.2" 24 | dc = "eqdc10" 25 | 26 | -------------------------------------------------------------------------------- /tests/data/test.yaml: -------------------------------------------------------------------------------- 1 | # Borrowed from Wikipedia: https://en.wikipedia.org/wiki/YAML#Sample_document 2 | --- 3 | receipt: Oz-Ware Purchase Invoice 4 | date: 2012-08-06 5 | customer: 6 | first_name: Dorothy 7 | family_name: Gale 8 | 9 | items: 10 | - part_no: A4786 11 | descrip: Water Bucket (Filled) 12 | price: 1.47 13 | quantity: 4 14 | 15 | - part_no: E1628 16 | descrip: High Heeled "Ruby" Slippers 17 | size: 8 18 | price: 133.7 19 | quantity: 1 20 | 21 | bill-to: &id001 22 | street: | 23 | 123 Tornado Alley 24 | Suite 16 25 | city: East Centerville 26 | state: KS 27 | 28 | ship-to: *id001 29 | 30 | specialDelivery: > 31 | Follow the Yellow Brick 32 | Road to the Emerald City. 33 | Pay no attention to the 34 | man behind the curtain. 35 | ... 36 | -------------------------------------------------------------------------------- /tests/data/uk_marriage_rates_2022.xslx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HariSekhon/DevOps-Python-tools/add60aa8705f2481287cf0fe9b77c0aa7b4e78fd/tests/data/uk_marriage_rates_2022.xslx -------------------------------------------------------------------------------- /tests/docker/apache-drill-docker-compose.yml: -------------------------------------------------------------------------------- 1 | # vim:ts=2:sts=2:sw=2:et 2 | # 3 | # Author: Hari Sekhon 4 | # Date: 2016-12-09 16:18:36 +0000 (Fri, 09 Dec 2016) 5 | # 6 | # https://github.com/HariSekhon/DevOps-Python-tools 7 | # 8 | # License: see accompanying Hari Sekhon LICENSE file 9 | # 10 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 11 | # 12 | # https://www.linkedin.com/in/HariSekhon 13 | # 14 | 15 | version: '2.2' 16 | services: 17 | apache-drill: 18 | extends: 19 | file: common.yml 20 | service: common 21 | image: harisekhon/apache-drill:${VERSION:-latest} 22 | ports: 23 | - 8047 24 | command: ['supervisord', '-n'] 25 | links: 26 | - apache-drill-zookeeper:zookeeper 27 | networks: 28 | - apache_drill 29 | apache-drill-zookeeper: 30 | extends: 31 | file: common.yml 32 | service: common 33 | image: harisekhon/zookeeper:latest 34 | ports: 35 | - 2181 36 | - 3181 37 | - 4181 38 | networks: 39 | - apache_drill 40 | 41 | networks: 42 | apache_drill: 43 | -------------------------------------------------------------------------------- /tests/docker/common.yml: -------------------------------------------------------------------------------- 1 | # vim:ts=2:sts=2:sw=2:et 2 | # 3 | # Author: Hari Sekhon 4 | # Date: 2016-12-09 15:16:43 +0000 (Fri, 09 Dec 2016) 5 | # 6 | # https://github.com/HariSekhon/DevOps-Python-tools 7 | # 8 | # License: see accompanying Hari Sekhon LICENSE file 9 | # 10 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 11 | # 12 | # https://www.linkedin.com/in/HariSekhon 13 | # 14 | 15 | version: '2.2' 16 | services: 17 | common: 18 | environment: 19 | - DEBUG 20 | volumes: 21 | - ../..:/devops-python-tools 22 | -------------------------------------------------------------------------------- /tests/docker/elasticsearch-docker-compose.yml: -------------------------------------------------------------------------------- 1 | # vim:ts=2:sts=2:sw=2:et 2 | # 3 | # Author: Hari Sekhon 4 | # Date: 2016-12-09 18:41:13 +0000 (Fri, 09 Dec 2016) 5 | # 6 | # https://github.com/HariSekhon/DevOps-Python-tools 7 | # 8 | # License: see accompanying Hari Sekhon LICENSE file 9 | # 10 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 11 | # 12 | # https://www.linkedin.com/in/HariSekhon 13 | # 14 | 15 | version: '2.2' 16 | services: 17 | elasticsearch: 18 | extends: 19 | file: common.yml 20 | service: common 21 | image: elasticsearch:${VERSION:-latest} 22 | ports: 23 | - 9200 24 | networks: 25 | - elasticsearch 26 | 27 | networks: 28 | elasticsearch: 29 | -------------------------------------------------------------------------------- /tests/docker/elasticsearch-elastic.co-docker-compose.yml: -------------------------------------------------------------------------------- 1 | # vim:ts=2:sts=2:sw=2:et 2 | # 3 | # Author: Hari Sekhon 4 | # Date: 2016-12-09 18:41:13 +0000 (Fri, 09 Dec 2016) 5 | # 6 | # https://github.com/HariSekhon/DevOps-Python-tools 7 | # 8 | # License: see accompanying Hari Sekhon LICENSE file 9 | # 10 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 11 | # 12 | # https://www.linkedin.com/in/HariSekhon 13 | # 14 | 15 | version: '2.2' 16 | services: 17 | elasticsearch: 18 | extends: 19 | file: common.yml 20 | service: common 21 | # Official Elasticsearch docker images have moved from DockerHub to docker.elastic.co for Elasticsearch 6.x + 22 | # elastic.co don't support 'latest' tag yet, nor x.y versions, so default to x.y.z for now 23 | image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION:-6.1.1} 24 | ports: 25 | - 9200 26 | networks: 27 | - elasticsearch 28 | 29 | networks: 30 | elasticsearch: 31 | -------------------------------------------------------------------------------- /tests/docker/hadoop-docker-compose.yml: -------------------------------------------------------------------------------- 1 | # vim:ts=2:sts=2:sw=2:et 2 | # 3 | # Author: Hari Sekhon 4 | # Date: 2016-12-09 21:25:07 +0000 (Fri, 09 Dec 2016) 5 | # 6 | # https://github.com/HariSekhon/DevOps-Python-tools 7 | # 8 | # License: see accompanying Hari Sekhon LICENSE file 9 | # 10 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 11 | # 12 | # https://www.linkedin.com/in/HariSekhon 13 | # 14 | 15 | version: '2.2' 16 | services: 17 | hadoop: 18 | extends: 19 | file: common.yml 20 | service: common 21 | image: harisekhon/hadoop-dev:${VERSION:-latest} 22 | ports: 23 | #- 8020 24 | - 8042 25 | - 8088 26 | #- 9000 27 | #- 10020 28 | - 19888 29 | - 50010 30 | - 50020 31 | - 50070 32 | - 50075 33 | - 50090 34 | volumes: 35 | - ../..:/py 36 | networks: 37 | - hadoop 38 | 39 | networks: 40 | hadoop: 41 | -------------------------------------------------------------------------------- /tests/docker/hbase-docker-compose.yml: -------------------------------------------------------------------------------- 1 | # vim:ts=2:sts=2:sw=2:et 2 | # 3 | # Author: Hari Sekhon 4 | # Date: 2016-12-09 22:13:19 +0000 (Fri, 09 Dec 2016) 5 | # 6 | # https://github.com/HariSekhon/DevOps-Python-tools 7 | # 8 | # License: see accompanying Hari Sekhon LICENSE file 9 | # 10 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 11 | # 12 | # https://www.linkedin.com/in/HariSekhon 13 | # 14 | 15 | version: '2.2' 16 | services: 17 | hbase: 18 | extends: 19 | file: common.yml 20 | service: common 21 | image: harisekhon/hbase-dev:${VERSION:-latest} 22 | ports: 23 | - 2181 24 | - 8080 25 | - 8085 26 | - 9090 27 | - 9095 28 | - 16000 29 | - 16010 30 | - 16030 # for HBase 0.99 31 | - 16201 32 | - 16301 33 | # older ports for HBase 0.96 34 | - 60000 35 | - 60010 36 | - 60201 37 | - 60301 38 | networks: 39 | - hbase 40 | 41 | networks: 42 | hbase: 43 | -------------------------------------------------------------------------------- /tests/docker/presto-dev-docker-compose.yml: -------------------------------------------------------------------------------- 1 | # vim:ts=2:sts=2:sw=2:et 2 | # 3 | # Author: Hari Sekhon 4 | # Date: 2017-09-13 14:47:23 +0200 (Wed, 13 Sep 2017) 5 | # 6 | # https://github.com/HariSekhon/DevOps-Python-tools 7 | # 8 | # License: see accompanying Hari Sekhon LICENSE file 9 | # 10 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 11 | # 12 | # https://www.linkedin.com/in/HariSekhon 13 | # 14 | 15 | version: '2.1' 16 | services: 17 | presto: 18 | image: harisekhon/presto-dev:${VERSION:-latest} 19 | ports: 20 | - 8080 21 | - 8081 22 | networks: 23 | - presto_dev 24 | 25 | networks: 26 | presto_dev: 27 | -------------------------------------------------------------------------------- /tests/docker/presto-docker-compose.yml: -------------------------------------------------------------------------------- 1 | # vim:ts=2:sts=2:sw=2:et 2 | # 3 | # Author: Hari Sekhon 4 | # Date: 2017-09-13 14:47:23 +0200 (Wed, 13 Sep 2017) 5 | # 6 | # https://github.com/HariSekhon/DevOps-Python-tools 7 | # 8 | # License: see accompanying Hari Sekhon LICENSE file 9 | # 10 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 11 | # 12 | # https://www.linkedin.com/in/HariSekhon 13 | # 14 | 15 | version: '2.1' 16 | services: 17 | presto: 18 | image: harisekhon/presto:${VERSION:-latest} 19 | ports: 20 | - 8080 21 | - 8081 22 | networks: 23 | - presto 24 | 25 | networks: 26 | presto: 27 | -------------------------------------------------------------------------------- /tests/docker/registry-docker-compose.yml: -------------------------------------------------------------------------------- 1 | # vim:ts=2:sts=2:sw=2:et 2 | # 3 | # Author: Hari Sekhon 4 | # Date: 2017-09-12 17:27:50 +0200 (Tue, 12 Sep 2017) 5 | # 6 | # https://github.com/HariSekhon/DevOps-Python-tools 7 | # 8 | # License: see accompanying Hari Sekhon LICENSE file 9 | # 10 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 11 | # 12 | # https://www.linkedin.com/in/HariSekhon 13 | # 14 | 15 | version: '2.2' 16 | 17 | services: 18 | pytools-registry-test: 19 | image: registry:2 20 | ports: 21 | - 5000:5000 22 | environment: 23 | - REGISTRY_HTTP_ADDR=0.0.0.0:5000 24 | - REGISTRY_HTTP_TLS_CERTIFICATE=/certs/registry.crt 25 | - REGISTRY_HTTP_TLS_KEY=/certs/registry.key 26 | - REGISTRY_AUTH_HTPASSWD_PATH=/certs/registry.htpasswd 27 | - REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm 28 | volumes: 29 | - .:/certs 30 | networks: 31 | - docker_registry 32 | 33 | networks: 34 | docker_registry: 35 | -------------------------------------------------------------------------------- /tests/docker/solrcloud-docker-compose.yml: -------------------------------------------------------------------------------- 1 | # vim:ts=2:sts=2:sw=2:et 2 | # 3 | # Author: Hari Sekhon 4 | # Date: 2016-12-13 14:01:04 +0000 (Tue, 13 Dec 2016) 5 | # 6 | # https://github.com/HariSekhon/DevOps-Python-tools 7 | # 8 | # License: see accompanying Hari Sekhon LICENSE file 9 | # 10 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 11 | # 12 | # https://www.linkedin.com/in/HariSekhon 13 | # 14 | 15 | version: '2.2' 16 | services: 17 | solrcloud: 18 | extends: 19 | file: common.yml 20 | service: common 21 | # using my own Docker images now as official solr doesn't have builds < 5 22 | user: solr 23 | image: harisekhon/solrcloud-dev:${VERSION:-latest} 24 | ports: 25 | - 8983 26 | - 8984 27 | - 9983 28 | networks: 29 | - solrcloud 30 | 31 | networks: 32 | solrcloud: 33 | -------------------------------------------------------------------------------- /tests/excluded.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | # intended only to be sourced by utils.sh 17 | # 18 | # split from utils.sh as this is specific to this repo 19 | 20 | set -eu 21 | [ -n "${DEBUG:-}" ] && set -x 22 | 23 | isExcluded(){ 24 | local prog="$1" 25 | # shellcheck disable=SC2049 26 | [[ "$prog" =~ ^\* ]] && return 0 27 | [[ "$prog" =~ spark_.*\.py ]] && return 0 28 | [[ "$prog" =~ \.jy ]] && return 0 29 | [[ "$prog" =~ hdfs_find_replication_factor_1\.py ]] && return 0 # python-krbV doesn't build on Python 3 30 | #[[ $prog =~ ipython-notebook ]] && return 0 31 | # this external git check is expensive, skip it when in CI as using fresh git checkouts 32 | is_CI && return 1 33 | # we no longer include spark inside pytools docker image due to bloat, so skip tests 34 | if is_inside_docker && [[ "$prog" =~ spark_* ]]; then 35 | return 0 36 | fi 37 | if type -P git &>/dev/null; then 38 | commit="$(git log "$prog" | head -n1 | grep 'commit')" 39 | if [ -z "$commit" ]; then 40 | return 0 41 | fi 42 | fi 43 | return 1 44 | } 45 | -------------------------------------------------------------------------------- /tests/help.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -eu 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 19 | 20 | cd "$srcdir/.."; 21 | 22 | # shellcheck disable=SC1091 23 | . ./tests/utils.sh 24 | 25 | # shellcheck disable=SC2068 26 | for x in ${@:-$(echo ./*.py 2>/dev/null)}; do 27 | isExcluded "$x" && continue 28 | set +e 29 | echo "$x --help" 30 | "$x" --help # >/dev/null 31 | status=$? 32 | set -e 33 | echo; hr 34 | if [ $status = 0 ]; then 35 | [[ "$x" =~ ambari_blueprints.py$ ]] && continue 36 | [[ "$x" =~ pythonpath.py$ ]] && continue 37 | [[ "$x" =~ aws_s3_presign.py$ ]] && continue 38 | elif [ $status = 1 ]; then 39 | if [[ "$x" =~ hdfs_find_replication_factor_1.py$ ]] && 40 | ! python -c 'import krbV'; then # best effort, not available on Mac any more 41 | continue 42 | fi 43 | fi 44 | [ $status = 3 ] || { echo "status code for $x --help was $status not expected 3"; exit 1; } 45 | done 46 | echo "All Python programs found exited with expected code 0/3 for --help" 47 | -------------------------------------------------------------------------------- /tests/python3.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -eu 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 19 | 20 | cd "$srcdir/.."; 21 | 22 | # shellcheck disable=SC1091 23 | . ./tests/utils.sh 24 | 25 | pip install caniusepython3 26 | echo "testing module dependencies for Python 3 compatibility" 27 | caniusepython3 -r requirements.txt 28 | -------------------------------------------------------------------------------- /tests/syntax.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -eu 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 19 | 20 | cd "$srcdir/.."; 21 | 22 | # shellcheck disable=SC1091 23 | . ./tests/utils.sh 24 | 25 | while read -r prog; do 26 | isExcluded "$prog" && continue 27 | if type -P flake8 &> /dev/null; then 28 | echo "flake8 $prog" 29 | flake8 --max-line-length=120 --statistics "$prog" 30 | echo; hr; echo 31 | fi 32 | for y in pyflakes pychecker; do 33 | if type -P "$y" &>/dev/null; then 34 | echo "$y $prog" 35 | "$y" "$prog" 36 | echo; hr; echo 37 | fi 38 | done 39 | done < <(find . -type f -name '*.py' -o -type f -name '*.jy') 40 | -------------------------------------------------------------------------------- /tests/test_docker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2016-12-08 14:38:37 +0000 (Thu, 08 Dec 2016) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$(cd "$(dirname "$0")" && pwd)" 19 | 20 | cd "$srcdir/.." 21 | 22 | # shellcheck disable=SC1091 23 | . "bash-tools/lib/docker.sh" 24 | 25 | # shellcheck disable=SC1091 26 | . "bash-tools/lib/utils.sh" 27 | 28 | section "Docker Image" 29 | 30 | export DOCKER_IMAGE="harisekhon/pytools" 31 | 32 | if is_docker_available; then 33 | [ -n "${NO_DOCKER:-}" ] && exit 0 34 | [ -n "${NO_PULL:-}" ] || 35 | docker pull "$DOCKER_IMAGE" 36 | docker run --rm "$DOCKER_IMAGE" welcome.py 37 | hr 38 | # this is too heavy and tests/all.sh is run as part of build 39 | docker run --rm -e DEBUG="${DEBUG:-}" "$DOCKER_IMAGE" tests/all.sh 40 | fi 41 | -------------------------------------------------------------------------------- /tests/test_dockerfiles_check_git_branches.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 19 | 20 | cd "$srcdir/.." 21 | 22 | # shellcheck disable=SC1091 23 | . "tests/utils.sh" 24 | 25 | # shellcheck disable=SC1091 26 | . "bash-tools/lib/utils.sh" 27 | 28 | section "Testing Dockerfiles Check Git branches" 29 | 30 | if type -P git &>dev/null; then 31 | if ! [ -d Dockerfiles ]; then 32 | git clone https://github.com/HariSekhon/Dockerfiles 33 | else 34 | pushd Dockerfiles 35 | git pull 36 | popd 37 | fi 38 | fi 39 | if [ -d Dockerfiles ]; then 40 | check './dockerfiles_check_git_branches.py Dockerfiles' "Dockerfiles Check Git branches of submodule Dockerfiles/" 41 | else 42 | echo "WARNING: Dockerfiles not present and git command not found to clone from github, skipping test" 43 | fi 44 | 45 | echo 46 | echo 47 | -------------------------------------------------------------------------------- /tests/test_dockerfiles_check_git_tags.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 19 | 20 | cd "$srcdir/.." 21 | 22 | # shellcheck disable=SC1091 23 | . "tests/utils.sh" 24 | 25 | # shellcheck disable=SC1091 26 | . "bash-tools/lib/utils.sh" 27 | 28 | section "Testing Dockerfiles Check Git Tags" 29 | 30 | if type -P git &>dev/null; then 31 | if ! [ -d Dockerfiles ]; then 32 | git clone https://github.com/HariSekhon/Dockerfiles 33 | else 34 | pushd Dockerfiles 35 | git pull 36 | popd 37 | fi 38 | fi 39 | if [ -d Dockerfiles ]; then 40 | check './dockerfiles_check_git_tags.py Dockerfiles' "Dockerfiles Check Git Tags of submodule Dockerfiles/" 41 | else 42 | echo "WARNING: Dockerfiles not present and git command not found to clone from github, skipping test" 43 | fi 44 | 45 | echo 46 | echo 47 | -------------------------------------------------------------------------------- /tests/test_dockerhub_search.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 19 | 20 | cd "$srcdir/.." 21 | 22 | # shellcheck disable=SC1091 23 | . "tests/utils.sh" 24 | 25 | # shellcheck disable=SC1091 26 | . "bash-tools/lib/utils.sh" 27 | 28 | section "Testing DockerHub Show Tags" 29 | 30 | check './dockerhub_search.py centos' "DockerHub Search for CentOS" 31 | check './dockerhub_search.py harisekhon' "DockerHub Search for harisekhon" 32 | check './dockerhub_search.py harisekhon -l 30' "DockerHub Search for harisekhon -l 30" 33 | # this no longer works, API must have changed 34 | #check './dockerhub_search.py harisekhon/hadoop-dev | grep harisekhon/hadoop-dev' "DockerHub Search for harisekhon/hadoop-dev" 35 | check './dockerhub_search.py hadoop-dev | grep harisekhon/hadoop-dev' "DockerHub Search for harisekhon/hadoop-dev" 36 | # causes IOError: [Errno 32] Broken pipe 37 | #unset PYTHONUNBUFFERED 38 | # shellcheck disable=SC2016 39 | check '[ "$(./dockerhub_search.py -q harisekhon | head -n 40 | tee /dev/stderr | grep -c "^harisekhon/[A-Za-z0-9_-]*$")" = 40 ]' "DockerHub Search quiet mode for shell scripting" 40 | 41 | echo 42 | echo 43 | -------------------------------------------------------------------------------- /tests/test_dockerhub_show_tags.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 19 | 20 | cd "$srcdir/.." 21 | 22 | # shellcheck disable=SC1091 23 | . "tests/utils.sh" 24 | 25 | # shellcheck disable=SC1091 26 | . "bash-tools/lib/utils.sh" 27 | 28 | section "Testing DockerHub Show Tags" 29 | 30 | start_time="$(start_timer "DockerHub Show Tags")" 31 | 32 | run ./dockerhub_show_tags.py centos debian 33 | run ./dockerhub_show_tags.py harisekhon/centos-java 34 | run_grep '6.7' ./dockerhub_show_tags.py centos 35 | 36 | run_grep "^latest$" ./dockerhub_show_tags.py -q centos 37 | 38 | run_grep "^centos6.9$" ./dockerhub_show_tags.py -q centos 39 | 40 | echo 41 | echo 42 | echo "All DockerHub Show Tags tests completed successfully" 43 | echo 44 | # $run_count defined in lib 45 | # shellcheck disable=SC2154 46 | echo "Total Tests run: $run_count" 47 | time_taken "$start_time" "DockerHub Show Tags tests completed in" 48 | echo 49 | echo 50 | -------------------------------------------------------------------------------- /tests/test_find_python_library_path.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2019-09-27 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -eu 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 19 | 20 | cd "$srcdir/.."; 21 | 22 | # shellcheck disable=SC1091 23 | . ./tests/utils.sh 24 | 25 | section "Python Find Library Path" 26 | 27 | start_time="$(start_timer "python_find_library_path")" 28 | 29 | run ./pythonpath.py 30 | 31 | run_grep '/os.py[co]?$' ./python_find_library_path.py os 32 | run_grep '/logging/__init__.py[co]?$' ./python_find_library_path.py logging 33 | ERRCODE=2 run_grep '' ./python_find_library_path.py nonexistentmodule 34 | run_grep 'python' ./python_find_library_path.py sys 35 | run_grep 'python' ./python_find_library_path.py 36 | run_usage ./python_find_library_path.py - 37 | run_usage ./python_find_library_path.py --help 38 | 39 | echo 40 | # shellcheck disable=SC2154 41 | echo "Total Tests run: $run_count" 42 | time_taken "$start_time" "SUCCESS! All tests for python_find_library_path.py completed in" 43 | echo 44 | -------------------------------------------------------------------------------- /tests/test_getent.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2016-11-20 15:35:37 +0000 (Sun, 20 Nov 2016) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$(cd "$(dirname "$0")" && pwd)" 19 | 20 | cd "$srcdir/.." 21 | 22 | # shellcheck disable=SC1091 23 | . ./bash-tools/lib/utils.sh 24 | 25 | section "Getent" 26 | 27 | start_time="$(start_timer "find_active_server.py test")" 28 | 29 | system="$(uname -s)" 30 | echo "system = $system" 31 | hr 32 | 33 | if [ "$system" = "Linux" ] || 34 | [ "$system" = Darwin ]; then 35 | run ./getent.py passwd | grep -v ':[x*!]*:' 36 | # counter is lost in subshell, increment manually 37 | run++ 38 | 39 | # $USER isn't always available in docker containers, use 'id' instead 40 | run ./getent.py passwd "$(id -un)" 41 | 42 | run_fail 2 ./getent.py passwd nonexistentuser 43 | 44 | run ./getent.py group | grep -v ':[x*!]:' 45 | run++ 46 | 47 | run ./getent.py group "$(id -gn)" 48 | 49 | run_fail 2 ./getent.py group nonexistentgroup 50 | 51 | echo 52 | # $run_count defined in lib 53 | # shellcheck disable=SC2154 54 | echo "Tests run: $run_count" 55 | time_taken "$start_time" "find_active_server.py tests completed in" 56 | else 57 | echo "system is not Linux or Mac, skipping..." 58 | fi 59 | -------------------------------------------------------------------------------- /tests/test_git_check_branches_upstream.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 19 | 20 | cd "$srcdir/.." 21 | 22 | # shellcheck disable=SC1091 23 | . "tests/utils.sh" 24 | 25 | # shellcheck disable=SC1091 26 | . "bash-tools/lib/utils.sh" 27 | 28 | section "Testing Git check branches upstream" 29 | 30 | if type -P git &>/dev/null; then 31 | if ! [ -d Dockerfiles ]; then 32 | git clone https://github.com/HariSekhon/Dockerfiles 33 | else 34 | pushd Dockerfiles 35 | git pull 36 | popd 37 | fi 38 | fi 39 | # TODO: check if this works later and re-enable 40 | # https://github.com/gitpython-developers/GitPython/issues/687 41 | if is_travis; then 42 | echo "WARNING: Skipping check on Travis CI due to bug in GitPython" 43 | exit 0 44 | fi 45 | if [ -d Dockerfiles ]; then 46 | check './git_check_branches_upstream.py . Dockerfiles' "Git check branches upstream of local repo and Dockerfiles submodule" 47 | else 48 | echo "WARNING: Dockerfiles not present and git command not found to clone from github, skipping test" 49 | fi 50 | 51 | echo 52 | echo 53 | -------------------------------------------------------------------------------- /tests/test_hexanonymize.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2020-01-02 17:35:08 +0000 (Thu, 02 Jan 2020) 6 | # 7 | # https://github.com/harisekhon/devop-python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | 12 | set -eu 13 | [ -n "${DEBUG:-}" ] && set -x 14 | srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 15 | 16 | cd "$srcdir/.."; 17 | 18 | # shellcheck disable=SC1091 19 | . ./tests/utils.sh 20 | 21 | section "HexAnonymize" 22 | 23 | start_time=$(date +%s) 24 | 25 | run++ 26 | check_output "abc123456def789012abcd" hexanonymize.py <<< "xyz987654rst654321AKIA" 27 | 28 | run++ 29 | check_output "abc123456def789012ABCD" hexanonymize.py -c <<< "xyz987654rst654321AKIA" 30 | 31 | run++ 32 | check_output "xyz123456rst789012abC" hexanonymize.py -c -o <<< "xyz987654rst654321caD" 33 | 34 | run++ 35 | check_output "xyz123456rst789012abc" hexanonymize.py -o <<< "xyz987654rst654321caD" 36 | 37 | echo 38 | # $run_count defined in lib 39 | # shellcheck disable=SC2154 40 | echo "Total Tests run: $run_count" 41 | time_taken "$start_time" "All version tests for hexanonymize.py completed in" 42 | echo 43 | untrap 44 | -------------------------------------------------------------------------------- /tests/test_json.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -eu 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 19 | 20 | echo " 21 | # ================== # 22 | # Testing JSON Files 23 | # ================== # 24 | " 25 | 26 | cd "$srcdir/.."; 27 | 28 | # shellcheck disable=SC1091 29 | . ./tests/utils.sh 30 | 31 | until [ $# -lt 1 ]; do 32 | case $1 in 33 | -*) shift || : 34 | esac 35 | done 36 | 37 | # ignore multi-line json data file for spark testing 38 | for jsonFile in $(find "${1:-.}" -iname '*.json' | 39 | grep -v -e '/spark-.*-bin-hadoop.*/' \ 40 | -e 'multirecord.json' \ 41 | -e 'broken' \ 42 | -e 'error'); do 43 | echo "testing json file: $jsonFile" 44 | python -mjson.tool < "$jsonFile" > /dev/null 45 | done 46 | 47 | echo 48 | echo 49 | -------------------------------------------------------------------------------- /tests/test_json_to_xml.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2016-08-29 18:18:39 +0100 (Mon, 29 Aug 2016) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$(cd "$(dirname "$0")" && pwd)" 19 | 20 | cd "$srcdir"; 21 | 22 | # shellcheck disable=SC1091 23 | . utils.sh 24 | 25 | # shellcheck disable=SC1091 26 | . ../bash-tools/lib/utils.sh 27 | 28 | section "JSON => XML" 29 | 30 | #tmpfile="$(mktemp json_to_xml_test.XXXXX.xml)" 31 | #echo "tmpfile is $tmpfile" 32 | 33 | #trap "rm -f $tmpfile" $TRAP_SIGNALS 34 | 35 | echo "running json_to_xml.py:" 36 | ../json_to_xml.py data/test.json | tee /dev/stderr | validate_xml.py 37 | echo 38 | 39 | echo "running json_to_xml.py from stdin:" 40 | ../json_to_xml.py < data/test.json | tee /dev/stderr | validate_xml.py 41 | echo 42 | 43 | echo "JSON to XML tests succeeded!" 44 | echo 45 | -------------------------------------------------------------------------------- /tests/test_json_to_yaml.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2019-12-19 18:04:15 +0000 (Thu, 19 Dec 2019) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$(cd "$(dirname "$0")" && pwd)" 19 | 20 | cd "$srcdir/.."; 21 | 22 | # shellcheck disable=SC1091,SC1090 23 | . "$srcdir/utils.sh" 24 | 25 | # shellcheck disable=SC1091,SC1090 26 | . "$srcdir/../bash-tools/lib/utils.sh" 27 | 28 | section "JSON => YAML" 29 | 30 | tmpfile="$(mktemp json_to_yaml_test.XXXXX.yml)" 31 | #echo "tmpfile is $tmpfile" 32 | 33 | # want var splitting 34 | # shellcheck disable=SC2086 35 | trap 'rm -f "$tmpfile"' $TRAP_SIGNALS 36 | 37 | for x in ./cloudformation/centos7-12nodes-encrypted.json tests/data/embedded_double_quotes.json; do 38 | echo "running json_to_yaml.py $x" 39 | ./json_to_yaml.py "$x" > "$tmpfile" 40 | echo "now validating generated yaml" 41 | ./validate_yaml.py "$tmpfile" 42 | echo 43 | done 44 | 45 | echo "recursing directory to convert all json files under a directory tree to yaml" 46 | ./json_to_yaml.py cloudformation/ > "$tmpfile" 47 | # TODO: fix validate_yaml.py to work on multi-yamls with --- and re-enable 48 | #echo "now validating generated yaml" 49 | #./validate_yaml.py "$tmpfile" 50 | echo "Success" 51 | -------------------------------------------------------------------------------- /tests/test_quay_show_tags.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 19 | 20 | cd "$srcdir/.." 21 | 22 | # shellcheck disable=SC1091 23 | . "tests/utils.sh" 24 | 25 | # shellcheck disable=SC1091 26 | . "bash-tools/lib/utils.sh" 27 | 28 | section "Testing Quay.io Show Tags" 29 | 30 | start_time="$(start_timer "Quay.io Show Tags")" 31 | 32 | run ./quay_show_tags.py coreos/etcd 33 | run_grep 'v0.4.8' ./quay_show_tags.py coreos/etcd 34 | 35 | run_grep "^latest$" ./quay_show_tags.py -q coreos/etcd 36 | 37 | run_grep "^v0.4.6$" ./quay_show_tags.py -q coreos/etcd 38 | 39 | echo 40 | echo 41 | echo "All Quay Show Tags tests completed successfully" 42 | echo 43 | # $run_count defined in lib 44 | # shellcheck disable=SC2154 45 | echo "Total Tests run: $run_count" 46 | time_taken "$start_time" "Quay Show Tags tests completed in" 47 | echo 48 | echo 49 | -------------------------------------------------------------------------------- /tests/test_serf_event_handler.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 19 | 20 | cd "$srcdir/.." 21 | 22 | # shellcheck disable=SC1091 23 | . "tests/utils.sh" 24 | 25 | # shellcheck disable=SC1091 26 | . "bash-tools/lib/utils.sh" 27 | 28 | section "Testing Serf Event Handler" 29 | 30 | check 'SERF_EVENT="query" SERF_QUERY_NAME="uptime" ./serf_event_handler.py --cmd-passthru -D < /dev/null | grep "load average"' "Serf Event Handler" 31 | 32 | check 'echo "myData" | SERF_EVENT="query" SERF_QUERY_NAME="uptime" ./serf_event_handler.py --cmd-passthru -D 2>&1 | grep "data:[[:space:]]myData"' "Serf Event Handler with data" 33 | 34 | echo 35 | echo 36 | -------------------------------------------------------------------------------- /tests/test_spark_json_to_parquet.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -eu 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 19 | 20 | cd "$srcdir" 21 | 22 | # shellcheck disable=SC1091 23 | . ./utils.sh 24 | 25 | section "Spark JSON => Parquet" 26 | 27 | if is_inside_docker; then 28 | echo "detected running inside docker, skipping test..." 29 | return 0 &>/dev/null || : 30 | exit 0 31 | fi 32 | 33 | export SPARK_VERSIONS="${*:-1.3.1 1.4.0 1.5.1 1.6.2 2.0.0}" 34 | 35 | for SPARK_VERSION in $SPARK_VERSIONS; do 36 | dir="spark-$SPARK_VERSION-bin-hadoop2.6" 37 | tar="$dir.tgz" 38 | if ! [ -d "$dir" ]; then 39 | if ! [ -f "$tar" ]; then 40 | echo "fetching $tar" 41 | # some systems don't have wget 42 | if type -P wget &>/dev/null; then 43 | wget "http://d3kbcqa49mib13.cloudfront.net/$tar" 44 | else 45 | curl -L "http://d3kbcqa49mib13.cloudfront.net/$tar" > "$tar" 46 | fi 47 | fi 48 | echo "untarring $tar" 49 | tar zxf "$tar" || rm -f "$tar" "$dir" 50 | fi 51 | echo 52 | export SPARK_HOME="$dir" 53 | rm -fr "test-$dir.parquet" 54 | if ../spark_json_to_parquet.py -j data/multirecord.json -p "test-$dir.parquet"; then 55 | echo "SUCCEEDED with Spark $SPARK_VERSION" 56 | else 57 | echo "FAILED test with Spark $SPARK_VERSION" 58 | exit 1 59 | fi 60 | done 61 | echo "SUCCESS" 62 | -------------------------------------------------------------------------------- /tests/test_spark_z_avro_to_parquet.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -eu 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 19 | 20 | cd "$srcdir" 21 | 22 | # shellcheck disable=SC1091 23 | . ./utils.sh 24 | 25 | section "Spark Avro => Parquet" 26 | 27 | if is_inside_docker; then 28 | echo "detected running inside docker, skipping test..." 29 | return 0 &>/dev/null || : 30 | exit 0 31 | fi 32 | 33 | export SPARK_VERSIONS="${*:-1.4.0 1.5.1 1.6.2}" 34 | # requires using spark-avro 3.0.0+ 35 | #export SPARK_VERSIONS="${*:-2.0.0}" 36 | 37 | for SPARK_VERSION in $SPARK_VERSIONS; do 38 | dir="spark-$SPARK_VERSION-bin-hadoop2.6" 39 | tar="$dir.tgz" 40 | if ! [ -d "$dir" ]; then 41 | if ! [ -f "$tar" ]; then 42 | echo "fetching $tar" 43 | # some systems don't have wget 44 | if type -P wget &>/dev/null; then 45 | wget "http://d3kbcqa49mib13.cloudfront.net/$tar" 46 | else 47 | curl -L "http://d3kbcqa49mib13.cloudfront.net/$tar" > "$tar" 48 | fi 49 | fi 50 | echo "untarring $tar" 51 | tar zxf "$tar" || rm -f "$tar" "$dir" 52 | fi 53 | echo 54 | export SPARK_HOME="$dir" 55 | rm -fr "test-$dir.parquet" 56 | if ../spark_avro_to_parquet.py -a "test-header-$dir.avro" -p "test-$dir.parquet"; then 57 | echo "SUCCEEDED with Spark $SPARK_VERSION" 58 | else 59 | echo "FAILED test with Spark $SPARK_VERSION" 60 | exit 1 61 | fi 62 | done 63 | echo "SUCCESS" 64 | -------------------------------------------------------------------------------- /tests/test_spark_z_parquet_to_avro.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -eu 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 19 | 20 | cd "$srcdir" 21 | 22 | # shellcheck disable=SC1091 23 | . ./utils.sh 24 | 25 | section "Spark Parquet => Avro" 26 | 27 | if is_inside_docker; then 28 | echo "detected running inside docker, skipping test..." 29 | return 0 &>/dev/null || : 30 | exit 0 31 | fi 32 | 33 | export SPARK_VERSIONS="${*:-1.4.0 1.5.1 1.6.2}" 34 | # requires using spark-avro 3.0.0+ 35 | #export SPARK_VERSIONS="${*:-2.0.0}" 36 | 37 | for SPARK_VERSION in $SPARK_VERSIONS; do 38 | dir="spark-$SPARK_VERSION-bin-hadoop2.6" 39 | tar="$dir.tgz" 40 | if ! [ -d "$dir" ]; then 41 | if ! [ -f "$tar" ]; then 42 | echo "fetching $tar" 43 | # some systems don't have wget 44 | if type -P wget &>/dev/null; then 45 | wget "http://d3kbcqa49mib13.cloudfront.net/$tar" 46 | else 47 | curl -L "http://d3kbcqa49mib13.cloudfront.net/$tar" > "$tar" 48 | fi 49 | fi 50 | echo "untarring $tar" 51 | tar zxf "$tar" || rm -f "$tar" "$dir" 52 | fi 53 | echo 54 | export SPARK_HOME="$dir" 55 | rm -fr "test-$dir.avro" 56 | if ../spark_parquet_to_avro.py -p "test-$dir.parquet" -a "test-$dir.avro"; then 57 | echo "SUCCEEDED with Spark $SPARK_VERSION" 58 | else 59 | echo "FAILED test with Spark $SPARK_VERSION" 60 | exit 1 61 | fi 62 | done 63 | echo "SUCCESS" 64 | -------------------------------------------------------------------------------- /tests/test_strip_ansi_escape_codes.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2018-09-09 23:09:20 +0100 (Sun, 09 Sep 2018) 6 | # 7 | # https://github.com/harisekhon/devop-python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | 12 | set -eu 13 | [ -n "${DEBUG:-}" ] && set -x 14 | srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 15 | 16 | cd "$srcdir/.."; 17 | 18 | # shellcheck disable=SC1091 19 | . ./tests/utils.sh 20 | 21 | section "Strip ANSI Escape Codes" 22 | 23 | name="strip_ansi_escape_codes.py" 24 | 25 | start_time=$(date +%s) 26 | 27 | #if is_mac; then 28 | # cat_opts="-e" 29 | #else 30 | # cat_opts="-A" 31 | #fi 32 | run++ 33 | if echo "some highlighted content" | 34 | grep --color=yes highlighted | 35 | ./strip_ansi_escape_codes.py | 36 | tee /dev/stderr | 37 | grep -q '^some highlighted content$'; then 38 | echo "ANSI escape code stripping SUCCEEDED" 39 | else 40 | echo "ANSI escape code stripping FAILED" 41 | exit 1 42 | fi 43 | hr 44 | 45 | tmp=$(mktemp /tmp/strip_ansi_escape_codes.XXXXX) 46 | # shellcheck disable=SC2064,SC2086 47 | trap "rm $tmp" $TRAP_SIGNALS 48 | 49 | echo 50 | echo "creating tmp file:" 51 | echo "some highlighted content" | grep --color=yes highlighted > "$tmp" 52 | hr 53 | 54 | echo 55 | echo "checking stripping from file" 56 | run++ 57 | if ./strip_ansi_escape_codes.py "$tmp" | 58 | tee /dev/stderr | 59 | grep -q '^some highlighted content$'; then 60 | echo "ANSI escape code stripping SUCCEEDED" 61 | else 62 | echo "ANSI escape code stripping FAILED" 63 | exit 1 64 | fi 65 | 66 | echo 67 | # $run_count defined in lib 68 | # shellcheck disable=SC2154 69 | echo "Total Tests run: $run_count" 70 | time_taken "$start_time" "All version tests for $name completed in" 71 | echo 72 | untrap 73 | -------------------------------------------------------------------------------- /tests/test_timeout.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2016-02-14 16:16:00 +0000 (Sun, 14 Feb 2016) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 19 | 20 | # shellcheck disable=SC1090 21 | . "$srcdir/../bash-tools/lib/utils.sh" 22 | 23 | run_fail 3 "$srcdir/../timeout.py" -t 2 sleep 10 24 | -------------------------------------------------------------------------------- /tests/test_welcome.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -eu 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 19 | 20 | cd "$srcdir/.."; 21 | 22 | # shellcheck disable=SC1091 23 | . ./tests/utils.sh 24 | 25 | # Fedora doesn't have /var/log/wtmp 26 | if ! [ -f /var/log/wtmp ]; then 27 | echo "/var/log/wtmp doesn't exist, touching..." 28 | # assigned in utils.sh 29 | # shellcheck disable=SC2154 30 | $sudo touch /var/log/wtmp || : 31 | fi 32 | 33 | ./welcome.py "$@" 34 | -------------------------------------------------------------------------------- /tests/test_xml_to_json.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2016-08-29 18:18:39 +0100 (Mon, 29 Aug 2016) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 19 | 20 | cd "$srcdir" 21 | 22 | # shellcheck disable=SC1091,SC1090 23 | . "$srcdir/utils.sh" 24 | 25 | # shellcheck disable=SC1091,SC1090 26 | . "$srcdir/../bash-tools/lib/utils.sh" 27 | 28 | section "XML => JSON" 29 | 30 | cd "$srcdir/.." 31 | 32 | testdata="tests/data/simple.xml" 33 | 34 | echo "running xml_to_json.py on $testdata": 35 | ./xml_to_json.py "$testdata" | tee /dev/stderr | ./validate_json.py 36 | echo 37 | 38 | echo "running xml_to_json.py on stdin < $testdata": 39 | ./xml_to_json.py < "$testdata" | tee /dev/stderr | ./validate_json.py 40 | echo 41 | 42 | echo "running xml_to_json.py on tests/data/plant_catalog.xml": 43 | ./xml_to_json.py "tests/data/plant_catalog.xml" | ./validate_json.py 44 | echo 45 | echo "XML to JSON tests succeeded!" 46 | echo 47 | -------------------------------------------------------------------------------- /tests/test_xml_to_yaml.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2016-08-29 18:18:39 +0100 (Mon, 29 Aug 2016) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 19 | 20 | cd "$srcdir" 21 | 22 | # shellcheck disable=SC1091,SC1090 23 | . "$srcdir/utils.sh" 24 | 25 | # shellcheck disable=SC1091,SC1090 26 | . "$srcdir/../bash-tools/lib/utils.sh" 27 | 28 | section "XML => YAML" 29 | 30 | cd "$srcdir/.." 31 | 32 | testdata="tests/data/simple.xml" 33 | 34 | echo "running xml_to_yaml.py on $testdata": 35 | ./xml_to_yaml.py "$testdata" | tee /dev/stderr | ./validate_yaml.py 36 | echo 37 | 38 | echo "running xml_to_yaml.py on stdin < $testdata": 39 | ./xml_to_yaml.py < "$testdata" | tee /dev/stderr | ./validate_yaml.py 40 | echo 41 | 42 | echo "running xml_to_yaml.py on tests/data/plant_catalog.xml": 43 | ./xml_to_yaml.py "tests/data/plant_catalog.xml" | ./validate_yaml.py 44 | echo 45 | echo "XML to yaml tests succeeded!" 46 | echo 47 | -------------------------------------------------------------------------------- /tests/test_yamllint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim:ts=4:sts=4:sw=4:et 3 | # 4 | # Author: Hari Sekhon 5 | # Date: 2019-02-26 14:40:53 +0000 (Tue, 26 Feb 2019) 6 | # 7 | # https://github.com/HariSekhon/DevOps-Python-tools 8 | # 9 | # License: see accompanying Hari Sekhon LICENSE file 10 | # 11 | # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish 12 | # 13 | # https://www.linkedin.com/in/HariSekhon 14 | # 15 | 16 | set -euo pipefail 17 | [ -n "${DEBUG:-}" ] && set -x 18 | srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 19 | 20 | cfg="$srcdir/../.yamllint" 21 | 22 | cd "$srcdir/.."; 23 | 24 | # shellcheck disable=SC1091 25 | . ./tests/utils.sh 26 | 27 | section "Running Yaml lint" 28 | 29 | yamllint -c "$cfg" . 30 | 31 | echo "=======" 32 | echo "SUCCESS" 33 | echo "=======" 34 | 35 | echo 36 | echo 37 | -------------------------------------------------------------------------------- /validate_properties.py: -------------------------------------------------------------------------------- 1 | validate_ini.py --------------------------------------------------------------------------------