├── .gitattributes ├── .github └── workflows │ ├── python-app.yml │ └── python-publish.yml ├── .gitignore ├── .pylintrc ├── .pylintrc.test ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── NOTICE ├── README.md ├── bin └── boundary-layer ├── boundary_layer ├── __init__.py ├── _version.py ├── builders │ ├── __init__.py │ ├── base.py │ ├── generator.py │ ├── primary.py │ ├── subdag.py │ ├── templates │ │ ├── generator_epilogue.j2 │ │ ├── generator_operator.j2 │ │ ├── generator_preamble.j2 │ │ ├── operator.j2 │ │ ├── primary_preamble.j2 │ │ ├── subdag_epilogue.j2 │ │ ├── subdag_operator.j2 │ │ └── subdag_preamble.j2 │ └── util.py ├── containers.py ├── exceptions.py ├── graph.py ├── logger.py ├── oozier │ ├── __init__.py │ ├── actions.py │ ├── cluster_config.py │ ├── file_fetcher.py │ ├── jsp_macros.py │ ├── parse.py │ └── schema.py ├── plugins │ ├── __init__.py │ ├── base.py │ ├── oozie_plugin.py │ ├── plugin_manager.py │ └── util.py ├── pretty_yaml.py ├── registry │ ├── __init__.py │ ├── registry.py │ └── types │ │ ├── __init__.py │ │ ├── generator.py │ │ ├── operator.py │ │ ├── preprocessor.py │ │ ├── resource.py │ │ └── subdag.py ├── schemas │ ├── __init__.py │ ├── base.py │ ├── dag.py │ └── internal │ │ ├── __init__.py │ │ ├── base.py │ │ ├── generators.py │ │ ├── operators.py │ │ ├── resources.py │ │ └── subdags.py ├── util.py ├── validator.py └── workflow.py ├── boundary_layer_default_plugin ├── __init__.py ├── config │ ├── generators │ │ ├── dict_generator.yaml │ │ ├── fixed.yaml │ │ ├── gcs_list_generator.yaml │ │ ├── list_generator.yaml │ │ ├── list_object_generator.yaml │ │ └── requests_json_generator.yaml │ ├── operators │ │ ├── base.yaml │ │ ├── base_sensor.yaml │ │ ├── bash.yaml │ │ ├── bigquery_create_table.yaml │ │ ├── bigquery_operator.yaml │ │ ├── bigquery_to_gcs.yaml │ │ ├── branch_python_operator.yaml │ │ ├── cloud_sql_base_operator.yaml │ │ ├── cloud_sql_export_operator.yaml │ │ ├── dataflow_template.yaml │ │ ├── dataproc_cluster_create.yaml │ │ ├── dataproc_cluster_delete.yaml │ │ ├── dataproc_hadoop.yaml │ │ ├── dataproc_pyspark.yaml │ │ ├── dataproc_spark.yaml │ │ ├── docker.yaml │ │ ├── dummy.yaml │ │ ├── email_operator.yaml │ │ ├── external_task_sensor.yaml │ │ ├── flow_control.yaml │ │ ├── gcp_pubsub_publish.yaml │ │ ├── gcp_pubsub_pull_sensor.yaml │ │ ├── gcs_delete_operator.yaml │ │ ├── gcs_object_sensor.yaml │ │ ├── gcs_prefix_sensor.yaml │ │ ├── gcs_to_bigquery.yaml │ │ ├── gcs_to_gcs.yaml │ │ ├── gcs_to_s3.yaml │ │ ├── gke_pod_operator.yaml │ │ ├── hive_partition_sensor.yaml │ │ ├── http_operator.yaml │ │ ├── kubernetes.yaml │ │ ├── ml_engine_batch_prediction.yaml │ │ ├── ml_engine_model.yaml │ │ ├── ml_engine_training.yaml │ │ ├── ml_engine_version.yaml │ │ ├── python_operator.yaml │ │ ├── short_circuit_operator.yaml │ │ ├── ssh.yaml │ │ └── trigger_dag_run.yaml │ ├── resources │ │ └── dataproc_cluster.yaml │ └── subdags │ │ └── subdag.yaml ├── oozie_actions.py ├── oozie_plugin.py ├── plugin.py └── preprocessors.py ├── examples └── readme_example.yaml ├── release.py ├── setup.cfg ├── setup.py ├── test ├── __init__.py ├── builders │ ├── __init__.py │ └── test_util.py ├── conftest.py ├── data │ ├── bad-dags │ │ ├── invalid-dataproc-cluster-name.yaml │ │ └── invalid_subdag_plugin_test.yaml │ ├── good-dags │ │ ├── generator_test.yaml │ │ ├── multi_generators.yaml │ │ ├── simple_dataproc_dag.yaml │ │ └── subdag_test.yaml │ ├── oozie-workflows │ │ └── example │ │ │ ├── README.md │ │ │ ├── conf │ │ │ └── wordcount-mr-config.xml │ │ │ ├── config-default.xml │ │ │ ├── coordinator │ │ │ ├── coord.properties │ │ │ └── coord.xml │ │ │ ├── job.properties │ │ │ ├── sub-workflows │ │ │ └── workflow.xml │ │ │ └── workflow.xml │ └── registry │ │ ├── invalid-bad-config │ │ └── operators │ │ │ └── base.yaml │ │ ├── invalid-duplicate-names │ │ └── operators │ │ │ ├── base.yaml │ │ │ └── base2.yaml │ │ └── valid │ │ └── operators │ │ ├── allow_extra_properties.yaml │ │ ├── base.yaml │ │ └── requires_preprocessors.yaml ├── default_plugin │ ├── test_plugin.py │ └── test_preprocessors.py ├── oozier │ ├── __init__.py │ ├── test_jsp_translator.py │ └── test_oozie_converter.py ├── plugins │ ├── __init__.py │ ├── test_plugin_config │ │ └── operators │ │ │ └── java.yaml │ └── test_plugin_manager.py ├── registry │ └── types │ │ └── test_operator_registry.py ├── test_generators.py ├── test_graph.py └── test_workflow.py ├── tox.ini └── versioneer.py /.gitattributes: -------------------------------------------------------------------------------- 1 | boundary_layer/_version.py export-subst 2 | -------------------------------------------------------------------------------- /.github/workflows/python-app.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/.github/workflows/python-app.yml -------------------------------------------------------------------------------- /.github/workflows/python-publish.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/.github/workflows/python-publish.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/.gitignore -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/.pylintrc -------------------------------------------------------------------------------- /.pylintrc.test: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/.pylintrc.test -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/LICENSE -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/MANIFEST.in -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | boundary-layer 2 | 3 | Copyright 2018 Etsy Inc. 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/README.md -------------------------------------------------------------------------------- /bin/boundary-layer: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/bin/boundary-layer -------------------------------------------------------------------------------- /boundary_layer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/__init__.py -------------------------------------------------------------------------------- /boundary_layer/_version.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/_version.py -------------------------------------------------------------------------------- /boundary_layer/builders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/builders/__init__.py -------------------------------------------------------------------------------- /boundary_layer/builders/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/builders/base.py -------------------------------------------------------------------------------- /boundary_layer/builders/generator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/builders/generator.py -------------------------------------------------------------------------------- /boundary_layer/builders/primary.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/builders/primary.py -------------------------------------------------------------------------------- /boundary_layer/builders/subdag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/builders/subdag.py -------------------------------------------------------------------------------- /boundary_layer/builders/templates/generator_epilogue.j2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/builders/templates/generator_epilogue.j2 -------------------------------------------------------------------------------- /boundary_layer/builders/templates/generator_operator.j2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/builders/templates/generator_operator.j2 -------------------------------------------------------------------------------- /boundary_layer/builders/templates/generator_preamble.j2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/builders/templates/generator_preamble.j2 -------------------------------------------------------------------------------- /boundary_layer/builders/templates/operator.j2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/builders/templates/operator.j2 -------------------------------------------------------------------------------- /boundary_layer/builders/templates/primary_preamble.j2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/builders/templates/primary_preamble.j2 -------------------------------------------------------------------------------- /boundary_layer/builders/templates/subdag_epilogue.j2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/builders/templates/subdag_epilogue.j2 -------------------------------------------------------------------------------- /boundary_layer/builders/templates/subdag_operator.j2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/builders/templates/subdag_operator.j2 -------------------------------------------------------------------------------- /boundary_layer/builders/templates/subdag_preamble.j2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/builders/templates/subdag_preamble.j2 -------------------------------------------------------------------------------- /boundary_layer/builders/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/builders/util.py -------------------------------------------------------------------------------- /boundary_layer/containers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/containers.py -------------------------------------------------------------------------------- /boundary_layer/exceptions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/exceptions.py -------------------------------------------------------------------------------- /boundary_layer/graph.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/graph.py -------------------------------------------------------------------------------- /boundary_layer/logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/logger.py -------------------------------------------------------------------------------- /boundary_layer/oozier/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/oozier/__init__.py -------------------------------------------------------------------------------- /boundary_layer/oozier/actions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/oozier/actions.py -------------------------------------------------------------------------------- /boundary_layer/oozier/cluster_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/oozier/cluster_config.py -------------------------------------------------------------------------------- /boundary_layer/oozier/file_fetcher.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/oozier/file_fetcher.py -------------------------------------------------------------------------------- /boundary_layer/oozier/jsp_macros.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/oozier/jsp_macros.py -------------------------------------------------------------------------------- /boundary_layer/oozier/parse.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/oozier/parse.py -------------------------------------------------------------------------------- /boundary_layer/oozier/schema.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/oozier/schema.py -------------------------------------------------------------------------------- /boundary_layer/plugins/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/plugins/__init__.py -------------------------------------------------------------------------------- /boundary_layer/plugins/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/plugins/base.py -------------------------------------------------------------------------------- /boundary_layer/plugins/oozie_plugin.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/plugins/oozie_plugin.py -------------------------------------------------------------------------------- /boundary_layer/plugins/plugin_manager.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/plugins/plugin_manager.py -------------------------------------------------------------------------------- /boundary_layer/plugins/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/plugins/util.py -------------------------------------------------------------------------------- /boundary_layer/pretty_yaml.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/pretty_yaml.py -------------------------------------------------------------------------------- /boundary_layer/registry/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/registry/__init__.py -------------------------------------------------------------------------------- /boundary_layer/registry/registry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/registry/registry.py -------------------------------------------------------------------------------- /boundary_layer/registry/types/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/registry/types/__init__.py -------------------------------------------------------------------------------- /boundary_layer/registry/types/generator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/registry/types/generator.py -------------------------------------------------------------------------------- /boundary_layer/registry/types/operator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/registry/types/operator.py -------------------------------------------------------------------------------- /boundary_layer/registry/types/preprocessor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/registry/types/preprocessor.py -------------------------------------------------------------------------------- /boundary_layer/registry/types/resource.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/registry/types/resource.py -------------------------------------------------------------------------------- /boundary_layer/registry/types/subdag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/registry/types/subdag.py -------------------------------------------------------------------------------- /boundary_layer/schemas/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/schemas/__init__.py -------------------------------------------------------------------------------- /boundary_layer/schemas/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/schemas/base.py -------------------------------------------------------------------------------- /boundary_layer/schemas/dag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/schemas/dag.py -------------------------------------------------------------------------------- /boundary_layer/schemas/internal/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/schemas/internal/__init__.py -------------------------------------------------------------------------------- /boundary_layer/schemas/internal/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/schemas/internal/base.py -------------------------------------------------------------------------------- /boundary_layer/schemas/internal/generators.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/schemas/internal/generators.py -------------------------------------------------------------------------------- /boundary_layer/schemas/internal/operators.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/schemas/internal/operators.py -------------------------------------------------------------------------------- /boundary_layer/schemas/internal/resources.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/schemas/internal/resources.py -------------------------------------------------------------------------------- /boundary_layer/schemas/internal/subdags.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/schemas/internal/subdags.py -------------------------------------------------------------------------------- /boundary_layer/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/util.py -------------------------------------------------------------------------------- /boundary_layer/validator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/validator.py -------------------------------------------------------------------------------- /boundary_layer/workflow.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer/workflow.py -------------------------------------------------------------------------------- /boundary_layer_default_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/__init__.py -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/generators/dict_generator.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/generators/dict_generator.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/generators/fixed.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/generators/fixed.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/generators/gcs_list_generator.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/generators/gcs_list_generator.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/generators/list_generator.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/generators/list_generator.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/generators/list_object_generator.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/generators/list_object_generator.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/generators/requests_json_generator.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/generators/requests_json_generator.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/base.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/base.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/base_sensor.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/base_sensor.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/bash.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/bash.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/bigquery_create_table.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/bigquery_create_table.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/bigquery_operator.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/bigquery_operator.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/bigquery_to_gcs.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/bigquery_to_gcs.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/branch_python_operator.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/branch_python_operator.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/cloud_sql_base_operator.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/cloud_sql_base_operator.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/cloud_sql_export_operator.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/cloud_sql_export_operator.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/dataflow_template.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/dataflow_template.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/dataproc_cluster_create.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/dataproc_cluster_create.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/dataproc_cluster_delete.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/dataproc_cluster_delete.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/dataproc_hadoop.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/dataproc_hadoop.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/dataproc_pyspark.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/dataproc_pyspark.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/dataproc_spark.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/dataproc_spark.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/docker.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/docker.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/dummy.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/dummy.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/email_operator.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/email_operator.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/external_task_sensor.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/external_task_sensor.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/flow_control.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/flow_control.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/gcp_pubsub_publish.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/gcp_pubsub_publish.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/gcp_pubsub_pull_sensor.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/gcp_pubsub_pull_sensor.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/gcs_delete_operator.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/gcs_delete_operator.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/gcs_object_sensor.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/gcs_object_sensor.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/gcs_prefix_sensor.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/gcs_prefix_sensor.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/gcs_to_bigquery.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/gcs_to_bigquery.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/gcs_to_gcs.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/gcs_to_gcs.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/gcs_to_s3.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/gcs_to_s3.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/gke_pod_operator.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/gke_pod_operator.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/hive_partition_sensor.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/hive_partition_sensor.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/http_operator.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/http_operator.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/kubernetes.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/kubernetes.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/ml_engine_batch_prediction.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/ml_engine_batch_prediction.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/ml_engine_model.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/ml_engine_model.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/ml_engine_training.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/ml_engine_training.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/ml_engine_version.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/ml_engine_version.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/python_operator.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/python_operator.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/short_circuit_operator.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/short_circuit_operator.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/ssh.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/ssh.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/operators/trigger_dag_run.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/operators/trigger_dag_run.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/resources/dataproc_cluster.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/resources/dataproc_cluster.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/config/subdags/subdag.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/config/subdags/subdag.yaml -------------------------------------------------------------------------------- /boundary_layer_default_plugin/oozie_actions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/oozie_actions.py -------------------------------------------------------------------------------- /boundary_layer_default_plugin/oozie_plugin.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/oozie_plugin.py -------------------------------------------------------------------------------- /boundary_layer_default_plugin/plugin.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/plugin.py -------------------------------------------------------------------------------- /boundary_layer_default_plugin/preprocessors.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/boundary_layer_default_plugin/preprocessors.py -------------------------------------------------------------------------------- /examples/readme_example.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/examples/readme_example.yaml -------------------------------------------------------------------------------- /release.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/release.py -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/setup.cfg -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/setup.py -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/builders/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/builders/test_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/builders/test_util.py -------------------------------------------------------------------------------- /test/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/conftest.py -------------------------------------------------------------------------------- /test/data/bad-dags/invalid-dataproc-cluster-name.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/data/bad-dags/invalid-dataproc-cluster-name.yaml -------------------------------------------------------------------------------- /test/data/bad-dags/invalid_subdag_plugin_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/data/bad-dags/invalid_subdag_plugin_test.yaml -------------------------------------------------------------------------------- /test/data/good-dags/generator_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/data/good-dags/generator_test.yaml -------------------------------------------------------------------------------- /test/data/good-dags/multi_generators.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/data/good-dags/multi_generators.yaml -------------------------------------------------------------------------------- /test/data/good-dags/simple_dataproc_dag.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/data/good-dags/simple_dataproc_dag.yaml -------------------------------------------------------------------------------- /test/data/good-dags/subdag_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/data/good-dags/subdag_test.yaml -------------------------------------------------------------------------------- /test/data/oozie-workflows/example/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/data/oozie-workflows/example/README.md -------------------------------------------------------------------------------- /test/data/oozie-workflows/example/conf/wordcount-mr-config.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/data/oozie-workflows/example/conf/wordcount-mr-config.xml -------------------------------------------------------------------------------- /test/data/oozie-workflows/example/config-default.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/data/oozie-workflows/example/config-default.xml -------------------------------------------------------------------------------- /test/data/oozie-workflows/example/coordinator/coord.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/data/oozie-workflows/example/coordinator/coord.properties -------------------------------------------------------------------------------- /test/data/oozie-workflows/example/coordinator/coord.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/data/oozie-workflows/example/coordinator/coord.xml -------------------------------------------------------------------------------- /test/data/oozie-workflows/example/job.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/data/oozie-workflows/example/job.properties -------------------------------------------------------------------------------- /test/data/oozie-workflows/example/sub-workflows/workflow.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/data/oozie-workflows/example/sub-workflows/workflow.xml -------------------------------------------------------------------------------- /test/data/oozie-workflows/example/workflow.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/data/oozie-workflows/example/workflow.xml -------------------------------------------------------------------------------- /test/data/registry/invalid-bad-config/operators/base.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/data/registry/invalid-bad-config/operators/base.yaml -------------------------------------------------------------------------------- /test/data/registry/invalid-duplicate-names/operators/base.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/data/registry/invalid-duplicate-names/operators/base.yaml -------------------------------------------------------------------------------- /test/data/registry/invalid-duplicate-names/operators/base2.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/data/registry/invalid-duplicate-names/operators/base2.yaml -------------------------------------------------------------------------------- /test/data/registry/valid/operators/allow_extra_properties.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/data/registry/valid/operators/allow_extra_properties.yaml -------------------------------------------------------------------------------- /test/data/registry/valid/operators/base.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/data/registry/valid/operators/base.yaml -------------------------------------------------------------------------------- /test/data/registry/valid/operators/requires_preprocessors.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/data/registry/valid/operators/requires_preprocessors.yaml -------------------------------------------------------------------------------- /test/default_plugin/test_plugin.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/default_plugin/test_plugin.py -------------------------------------------------------------------------------- /test/default_plugin/test_preprocessors.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/default_plugin/test_preprocessors.py -------------------------------------------------------------------------------- /test/oozier/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/oozier/test_jsp_translator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/oozier/test_jsp_translator.py -------------------------------------------------------------------------------- /test/oozier/test_oozie_converter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/oozier/test_oozie_converter.py -------------------------------------------------------------------------------- /test/plugins/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/plugins/test_plugin_config/operators/java.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/plugins/test_plugin_config/operators/java.yaml -------------------------------------------------------------------------------- /test/plugins/test_plugin_manager.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/plugins/test_plugin_manager.py -------------------------------------------------------------------------------- /test/registry/types/test_operator_registry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/registry/types/test_operator_registry.py -------------------------------------------------------------------------------- /test/test_generators.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/test_generators.py -------------------------------------------------------------------------------- /test/test_graph.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/test_graph.py -------------------------------------------------------------------------------- /test/test_workflow.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/test/test_workflow.py -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/tox.ini -------------------------------------------------------------------------------- /versioneer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etsy/boundary-layer/HEAD/versioneer.py --------------------------------------------------------------------------------