├── LICENSE ├── README.md └── app ├── app.yaml ├── backend.yaml ├── help ├── changelog.md ├── cloudhistory.md ├── examples.md ├── install.md ├── overview.md └── usage.md ├── index.yaml ├── lib ├── __init__.py └── crud │ ├── __init__.py │ ├── crud_handler.py │ ├── crud_handler_test.py │ ├── crud_model.py │ ├── crud_model_test.py │ ├── crud_test.py │ ├── crud_utils.py │ └── crud_utils_test.py ├── queue.yaml ├── src ├── __init__.py ├── auth.py ├── basetest.py ├── clients │ ├── __init__.py │ ├── bigquery.py │ ├── bigquery_test.py │ ├── computeengine.py │ ├── gcs.py │ ├── gcs_test.py │ ├── s3.py │ └── s3_test.py ├── csvmatchreplace │ ├── __init__.py │ ├── timestamp.py │ ├── timestamp_test.py │ ├── transform.py │ └── transform_test.py ├── hadoop │ ├── __init__.py │ ├── csv_transformer_mapper_tmpl.py │ ├── csv_transformer_mapper_tmpl_test.py │ ├── datastore.py │ ├── datastore_test.py │ ├── gce_api.py │ ├── gce_api_test.py │ ├── hadoop_cluster.py │ ├── hadoop_cluster_test.py │ ├── hadoop_csv_transformer.py │ └── hadoop_csv_transformer_test.py ├── handlers │ ├── __init__.py │ ├── basehandler.py │ ├── basehandler_test.py │ ├── helphandler.py │ ├── linthandler.py │ ├── runhandler.py │ ├── runhandler_test.py │ ├── variablehandler.py │ └── variablehandler_test.py ├── model │ ├── __init__.py │ ├── appconfig.py │ ├── appconfig_test.py │ ├── pipeline.py │ ├── pipeline_test.py │ ├── runstat.py │ ├── runstat_test.py │ ├── user.py │ └── user_test.py ├── pipelines │ ├── __init__.py │ ├── linter.py │ ├── linter_test.py │ ├── pipeline.py │ ├── pipelines.py │ ├── runner.py │ ├── runner_test.py │ ├── shardstage.py │ ├── shardstage_test.py │ ├── stages │ │ ├── __init__.py │ │ ├── bigqueryoutput.py │ │ ├── bigqueryoutput_test.py │ │ ├── csvmatchreplace.py │ │ ├── datastoreinput.py │ │ ├── gcedatatransformer.py │ │ ├── gcedatatransformer_test.py │ │ ├── gcedisksinput.py │ │ ├── gceinstancesinput.py │ │ ├── gcezoneoperationsinput.py │ │ ├── gcezoneoperationsinput_test.py │ │ ├── gcscompositor.py │ │ ├── gcsdeleter.py │ │ ├── gcsinput.py │ │ ├── gcsoutput.py │ │ ├── hadoopcsvmatchreplace.py │ │ ├── hadoopsetup.py │ │ ├── hadoopsetup_test.py │ │ ├── hadoopshutdown.py │ │ ├── hadoopshutdown_test.py │ │ ├── httpinput.py │ │ ├── httpinput_test.py │ │ ├── s3input.py │ │ ├── s3input_test.py │ │ └── testlogconfigstage.py │ └── testdata │ │ ├── __init__.py │ │ ├── csvmatchreplace.json │ │ ├── datstoreinput.json │ │ ├── datstoreinput2.json │ │ ├── datstoreinput3.json │ │ ├── gcscompositor.json │ │ ├── gcstogcs.json │ │ ├── httpinput.json │ │ └── httpinput10meg.json ├── server.py └── server_test.py └── static ├── app.css ├── components ├── app │ └── app.js ├── appconfig │ ├── appconfig.js │ └── appconfig.ng ├── butterbar │ ├── butterbar.js │ └── butterbar.ng ├── download │ └── download.js ├── dropupload │ └── dropupload.js ├── help │ ├── help.css │ ├── help.js │ ├── index.html │ └── stages.ng ├── layout │ └── layout.js ├── pipelineeditor │ ├── pipelineeditor.css │ ├── pipelineeditor.js │ └── pipelineeditor.ng ├── pipelinelist │ ├── pipelinelist.css │ ├── pipelinelist.js │ └── pipelinelist.ng ├── pipelines │ └── pipelines.js ├── upload │ └── upload.js ├── user │ └── user.js └── variables │ ├── variables.css │ ├── variables.js │ └── variables.ng ├── examples ├── cloud_history_pipeline.json ├── datastoreinput.json ├── gcstobigquery.json ├── hadoop_pipeline.json ├── hadoop_shutdown.json ├── hadoop_start.json ├── hadoop_transform.json ├── httpinput.json ├── languagecodes.csv └── operations_history_pipeline.json ├── hadoop_scripts ├── gcs_to_hdfs_mapper.sh ├── hadoop-1.2.1.patch ├── hdfs_to_gcs_mapper.sh ├── mapreduce__at__master.sh ├── rpc_daemon │ ├── __main__.py │ └── favicon.ico └── startup-script.sh ├── img ├── close-x.png ├── favicon.ico ├── general_black.png ├── help │ ├── cloud_history_arch.png │ ├── image_0.png │ ├── image_1.png │ ├── image_2.png │ ├── image_2_big.png │ ├── image_3.png │ ├── image_3_big.png │ ├── image_4.png │ └── image_4_big.png └── pipeline.png ├── index.html ├── init.js ├── jquery.layout-latest.min.js └── main.js /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/README.md -------------------------------------------------------------------------------- /app/app.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/app.yaml -------------------------------------------------------------------------------- /app/backend.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/backend.yaml -------------------------------------------------------------------------------- /app/help/changelog.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/help/changelog.md -------------------------------------------------------------------------------- /app/help/cloudhistory.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/help/cloudhistory.md -------------------------------------------------------------------------------- /app/help/examples.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/help/examples.md -------------------------------------------------------------------------------- /app/help/install.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/help/install.md -------------------------------------------------------------------------------- /app/help/overview.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/help/overview.md -------------------------------------------------------------------------------- /app/help/usage.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/help/usage.md -------------------------------------------------------------------------------- /app/index.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/index.yaml -------------------------------------------------------------------------------- /app/lib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/lib/crud/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/lib/crud/crud_handler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/lib/crud/crud_handler.py -------------------------------------------------------------------------------- /app/lib/crud/crud_handler_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/lib/crud/crud_handler_test.py -------------------------------------------------------------------------------- /app/lib/crud/crud_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/lib/crud/crud_model.py -------------------------------------------------------------------------------- /app/lib/crud/crud_model_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/lib/crud/crud_model_test.py -------------------------------------------------------------------------------- /app/lib/crud/crud_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/lib/crud/crud_test.py -------------------------------------------------------------------------------- /app/lib/crud/crud_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/lib/crud/crud_utils.py -------------------------------------------------------------------------------- /app/lib/crud/crud_utils_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/lib/crud/crud_utils_test.py -------------------------------------------------------------------------------- /app/queue.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/queue.yaml -------------------------------------------------------------------------------- /app/src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/src/auth.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/auth.py -------------------------------------------------------------------------------- /app/src/basetest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/basetest.py -------------------------------------------------------------------------------- /app/src/clients/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/src/clients/bigquery.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/clients/bigquery.py -------------------------------------------------------------------------------- /app/src/clients/bigquery_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/clients/bigquery_test.py -------------------------------------------------------------------------------- /app/src/clients/computeengine.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/clients/computeengine.py -------------------------------------------------------------------------------- /app/src/clients/gcs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/clients/gcs.py -------------------------------------------------------------------------------- /app/src/clients/gcs_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/clients/gcs_test.py -------------------------------------------------------------------------------- /app/src/clients/s3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/clients/s3.py -------------------------------------------------------------------------------- /app/src/clients/s3_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/clients/s3_test.py -------------------------------------------------------------------------------- /app/src/csvmatchreplace/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/src/csvmatchreplace/timestamp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/csvmatchreplace/timestamp.py -------------------------------------------------------------------------------- /app/src/csvmatchreplace/timestamp_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/csvmatchreplace/timestamp_test.py -------------------------------------------------------------------------------- /app/src/csvmatchreplace/transform.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/csvmatchreplace/transform.py -------------------------------------------------------------------------------- /app/src/csvmatchreplace/transform_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/csvmatchreplace/transform_test.py -------------------------------------------------------------------------------- /app/src/hadoop/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/src/hadoop/csv_transformer_mapper_tmpl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/hadoop/csv_transformer_mapper_tmpl.py -------------------------------------------------------------------------------- /app/src/hadoop/csv_transformer_mapper_tmpl_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/hadoop/csv_transformer_mapper_tmpl_test.py -------------------------------------------------------------------------------- /app/src/hadoop/datastore.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/hadoop/datastore.py -------------------------------------------------------------------------------- /app/src/hadoop/datastore_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/hadoop/datastore_test.py -------------------------------------------------------------------------------- /app/src/hadoop/gce_api.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/hadoop/gce_api.py -------------------------------------------------------------------------------- /app/src/hadoop/gce_api_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/hadoop/gce_api_test.py -------------------------------------------------------------------------------- /app/src/hadoop/hadoop_cluster.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/hadoop/hadoop_cluster.py -------------------------------------------------------------------------------- /app/src/hadoop/hadoop_cluster_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/hadoop/hadoop_cluster_test.py -------------------------------------------------------------------------------- /app/src/hadoop/hadoop_csv_transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/hadoop/hadoop_csv_transformer.py -------------------------------------------------------------------------------- /app/src/hadoop/hadoop_csv_transformer_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/hadoop/hadoop_csv_transformer_test.py -------------------------------------------------------------------------------- /app/src/handlers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/src/handlers/basehandler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/handlers/basehandler.py -------------------------------------------------------------------------------- /app/src/handlers/basehandler_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/handlers/basehandler_test.py -------------------------------------------------------------------------------- /app/src/handlers/helphandler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/handlers/helphandler.py -------------------------------------------------------------------------------- /app/src/handlers/linthandler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/handlers/linthandler.py -------------------------------------------------------------------------------- /app/src/handlers/runhandler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/handlers/runhandler.py -------------------------------------------------------------------------------- /app/src/handlers/runhandler_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/handlers/runhandler_test.py -------------------------------------------------------------------------------- /app/src/handlers/variablehandler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/handlers/variablehandler.py -------------------------------------------------------------------------------- /app/src/handlers/variablehandler_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/handlers/variablehandler_test.py -------------------------------------------------------------------------------- /app/src/model/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/src/model/appconfig.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/model/appconfig.py -------------------------------------------------------------------------------- /app/src/model/appconfig_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/model/appconfig_test.py -------------------------------------------------------------------------------- /app/src/model/pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/model/pipeline.py -------------------------------------------------------------------------------- /app/src/model/pipeline_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/model/pipeline_test.py -------------------------------------------------------------------------------- /app/src/model/runstat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/model/runstat.py -------------------------------------------------------------------------------- /app/src/model/runstat_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/model/runstat_test.py -------------------------------------------------------------------------------- /app/src/model/user.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/model/user.py -------------------------------------------------------------------------------- /app/src/model/user_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/model/user_test.py -------------------------------------------------------------------------------- /app/src/pipelines/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/src/pipelines/linter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/linter.py -------------------------------------------------------------------------------- /app/src/pipelines/linter_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/linter_test.py -------------------------------------------------------------------------------- /app/src/pipelines/pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/pipeline.py -------------------------------------------------------------------------------- /app/src/pipelines/pipelines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/pipelines.py -------------------------------------------------------------------------------- /app/src/pipelines/runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/runner.py -------------------------------------------------------------------------------- /app/src/pipelines/runner_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/runner_test.py -------------------------------------------------------------------------------- /app/src/pipelines/shardstage.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/shardstage.py -------------------------------------------------------------------------------- /app/src/pipelines/shardstage_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/shardstage_test.py -------------------------------------------------------------------------------- /app/src/pipelines/stages/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/src/pipelines/stages/bigqueryoutput.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/stages/bigqueryoutput.py -------------------------------------------------------------------------------- /app/src/pipelines/stages/bigqueryoutput_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/stages/bigqueryoutput_test.py -------------------------------------------------------------------------------- /app/src/pipelines/stages/csvmatchreplace.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/stages/csvmatchreplace.py -------------------------------------------------------------------------------- /app/src/pipelines/stages/datastoreinput.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/stages/datastoreinput.py -------------------------------------------------------------------------------- /app/src/pipelines/stages/gcedatatransformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/stages/gcedatatransformer.py -------------------------------------------------------------------------------- /app/src/pipelines/stages/gcedatatransformer_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/stages/gcedatatransformer_test.py -------------------------------------------------------------------------------- /app/src/pipelines/stages/gcedisksinput.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/stages/gcedisksinput.py -------------------------------------------------------------------------------- /app/src/pipelines/stages/gceinstancesinput.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/stages/gceinstancesinput.py -------------------------------------------------------------------------------- /app/src/pipelines/stages/gcezoneoperationsinput.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/stages/gcezoneoperationsinput.py -------------------------------------------------------------------------------- /app/src/pipelines/stages/gcezoneoperationsinput_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/stages/gcezoneoperationsinput_test.py -------------------------------------------------------------------------------- /app/src/pipelines/stages/gcscompositor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/stages/gcscompositor.py -------------------------------------------------------------------------------- /app/src/pipelines/stages/gcsdeleter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/stages/gcsdeleter.py -------------------------------------------------------------------------------- /app/src/pipelines/stages/gcsinput.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/stages/gcsinput.py -------------------------------------------------------------------------------- /app/src/pipelines/stages/gcsoutput.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/stages/gcsoutput.py -------------------------------------------------------------------------------- /app/src/pipelines/stages/hadoopcsvmatchreplace.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/stages/hadoopcsvmatchreplace.py -------------------------------------------------------------------------------- /app/src/pipelines/stages/hadoopsetup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/stages/hadoopsetup.py -------------------------------------------------------------------------------- /app/src/pipelines/stages/hadoopsetup_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/stages/hadoopsetup_test.py -------------------------------------------------------------------------------- /app/src/pipelines/stages/hadoopshutdown.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/stages/hadoopshutdown.py -------------------------------------------------------------------------------- /app/src/pipelines/stages/hadoopshutdown_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/stages/hadoopshutdown_test.py -------------------------------------------------------------------------------- /app/src/pipelines/stages/httpinput.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/stages/httpinput.py -------------------------------------------------------------------------------- /app/src/pipelines/stages/httpinput_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/stages/httpinput_test.py -------------------------------------------------------------------------------- /app/src/pipelines/stages/s3input.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/stages/s3input.py -------------------------------------------------------------------------------- /app/src/pipelines/stages/s3input_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/stages/s3input_test.py -------------------------------------------------------------------------------- /app/src/pipelines/stages/testlogconfigstage.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/stages/testlogconfigstage.py -------------------------------------------------------------------------------- /app/src/pipelines/testdata/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/src/pipelines/testdata/csvmatchreplace.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/testdata/csvmatchreplace.json -------------------------------------------------------------------------------- /app/src/pipelines/testdata/datstoreinput.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/testdata/datstoreinput.json -------------------------------------------------------------------------------- /app/src/pipelines/testdata/datstoreinput2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/testdata/datstoreinput2.json -------------------------------------------------------------------------------- /app/src/pipelines/testdata/datstoreinput3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/testdata/datstoreinput3.json -------------------------------------------------------------------------------- /app/src/pipelines/testdata/gcscompositor.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/testdata/gcscompositor.json -------------------------------------------------------------------------------- /app/src/pipelines/testdata/gcstogcs.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/testdata/gcstogcs.json -------------------------------------------------------------------------------- /app/src/pipelines/testdata/httpinput.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/testdata/httpinput.json -------------------------------------------------------------------------------- /app/src/pipelines/testdata/httpinput10meg.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/pipelines/testdata/httpinput10meg.json -------------------------------------------------------------------------------- /app/src/server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/server.py -------------------------------------------------------------------------------- /app/src/server_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/src/server_test.py -------------------------------------------------------------------------------- /app/static/app.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/app.css -------------------------------------------------------------------------------- /app/static/components/app/app.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/components/app/app.js -------------------------------------------------------------------------------- /app/static/components/appconfig/appconfig.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/components/appconfig/appconfig.js -------------------------------------------------------------------------------- /app/static/components/appconfig/appconfig.ng: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/components/appconfig/appconfig.ng -------------------------------------------------------------------------------- /app/static/components/butterbar/butterbar.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/components/butterbar/butterbar.js -------------------------------------------------------------------------------- /app/static/components/butterbar/butterbar.ng: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/components/butterbar/butterbar.ng -------------------------------------------------------------------------------- /app/static/components/download/download.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/components/download/download.js -------------------------------------------------------------------------------- /app/static/components/dropupload/dropupload.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/components/dropupload/dropupload.js -------------------------------------------------------------------------------- /app/static/components/help/help.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/components/help/help.css -------------------------------------------------------------------------------- /app/static/components/help/help.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/components/help/help.js -------------------------------------------------------------------------------- /app/static/components/help/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/components/help/index.html -------------------------------------------------------------------------------- /app/static/components/help/stages.ng: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/components/help/stages.ng -------------------------------------------------------------------------------- /app/static/components/layout/layout.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/components/layout/layout.js -------------------------------------------------------------------------------- /app/static/components/pipelineeditor/pipelineeditor.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/components/pipelineeditor/pipelineeditor.css -------------------------------------------------------------------------------- /app/static/components/pipelineeditor/pipelineeditor.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/components/pipelineeditor/pipelineeditor.js -------------------------------------------------------------------------------- /app/static/components/pipelineeditor/pipelineeditor.ng: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/components/pipelineeditor/pipelineeditor.ng -------------------------------------------------------------------------------- /app/static/components/pipelinelist/pipelinelist.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/components/pipelinelist/pipelinelist.css -------------------------------------------------------------------------------- /app/static/components/pipelinelist/pipelinelist.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/components/pipelinelist/pipelinelist.js -------------------------------------------------------------------------------- /app/static/components/pipelinelist/pipelinelist.ng: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/components/pipelinelist/pipelinelist.ng -------------------------------------------------------------------------------- /app/static/components/pipelines/pipelines.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/components/pipelines/pipelines.js -------------------------------------------------------------------------------- /app/static/components/upload/upload.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/components/upload/upload.js -------------------------------------------------------------------------------- /app/static/components/user/user.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/components/user/user.js -------------------------------------------------------------------------------- /app/static/components/variables/variables.css: -------------------------------------------------------------------------------- 1 | .variables-list { 2 | padding: 5px 20px; 3 | } 4 | -------------------------------------------------------------------------------- /app/static/components/variables/variables.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/components/variables/variables.js -------------------------------------------------------------------------------- /app/static/components/variables/variables.ng: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/components/variables/variables.ng -------------------------------------------------------------------------------- /app/static/examples/cloud_history_pipeline.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/examples/cloud_history_pipeline.json -------------------------------------------------------------------------------- /app/static/examples/datastoreinput.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/examples/datastoreinput.json -------------------------------------------------------------------------------- /app/static/examples/gcstobigquery.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/examples/gcstobigquery.json -------------------------------------------------------------------------------- /app/static/examples/hadoop_pipeline.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/examples/hadoop_pipeline.json -------------------------------------------------------------------------------- /app/static/examples/hadoop_shutdown.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/examples/hadoop_shutdown.json -------------------------------------------------------------------------------- /app/static/examples/hadoop_start.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/examples/hadoop_start.json -------------------------------------------------------------------------------- /app/static/examples/hadoop_transform.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/examples/hadoop_transform.json -------------------------------------------------------------------------------- /app/static/examples/httpinput.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/examples/httpinput.json -------------------------------------------------------------------------------- /app/static/examples/languagecodes.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/examples/languagecodes.csv -------------------------------------------------------------------------------- /app/static/examples/operations_history_pipeline.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/examples/operations_history_pipeline.json -------------------------------------------------------------------------------- /app/static/hadoop_scripts/gcs_to_hdfs_mapper.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/hadoop_scripts/gcs_to_hdfs_mapper.sh -------------------------------------------------------------------------------- /app/static/hadoop_scripts/hadoop-1.2.1.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/hadoop_scripts/hadoop-1.2.1.patch -------------------------------------------------------------------------------- /app/static/hadoop_scripts/hdfs_to_gcs_mapper.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/hadoop_scripts/hdfs_to_gcs_mapper.sh -------------------------------------------------------------------------------- /app/static/hadoop_scripts/mapreduce__at__master.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/hadoop_scripts/mapreduce__at__master.sh -------------------------------------------------------------------------------- /app/static/hadoop_scripts/rpc_daemon/__main__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/hadoop_scripts/rpc_daemon/__main__.py -------------------------------------------------------------------------------- /app/static/hadoop_scripts/rpc_daemon/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/hadoop_scripts/rpc_daemon/favicon.ico -------------------------------------------------------------------------------- /app/static/hadoop_scripts/startup-script.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/hadoop_scripts/startup-script.sh -------------------------------------------------------------------------------- /app/static/img/close-x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/img/close-x.png -------------------------------------------------------------------------------- /app/static/img/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/img/favicon.ico -------------------------------------------------------------------------------- /app/static/img/general_black.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/img/general_black.png -------------------------------------------------------------------------------- /app/static/img/help/cloud_history_arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/img/help/cloud_history_arch.png -------------------------------------------------------------------------------- /app/static/img/help/image_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/img/help/image_0.png -------------------------------------------------------------------------------- /app/static/img/help/image_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/img/help/image_1.png -------------------------------------------------------------------------------- /app/static/img/help/image_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/img/help/image_2.png -------------------------------------------------------------------------------- /app/static/img/help/image_2_big.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/img/help/image_2_big.png -------------------------------------------------------------------------------- /app/static/img/help/image_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/img/help/image_3.png -------------------------------------------------------------------------------- /app/static/img/help/image_3_big.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/img/help/image_3_big.png -------------------------------------------------------------------------------- /app/static/img/help/image_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/img/help/image_4.png -------------------------------------------------------------------------------- /app/static/img/help/image_4_big.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/img/help/image_4_big.png -------------------------------------------------------------------------------- /app/static/img/pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/img/pipeline.png -------------------------------------------------------------------------------- /app/static/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/index.html -------------------------------------------------------------------------------- /app/static/init.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/init.js -------------------------------------------------------------------------------- /app/static/jquery.layout-latest.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/jquery.layout-latest.min.js -------------------------------------------------------------------------------- /app/static/main.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/Data-Pipeline/HEAD/app/static/main.js --------------------------------------------------------------------------------