├── .gitignore ├── LICENSE ├── README.md ├── dags ├── __init__.py ├── bigquery.py ├── cloud_storage.py ├── dataproc.py ├── gcp_smoke │ └── gsob_extract_day.sql └── support │ ├── __init__.py │ └── schemas.py ├── external ├── afgc-dataflow-java │ ├── .gitignore │ ├── build.gradle │ ├── gradle │ │ └── wrapper │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ ├── gradlew │ ├── gradlew.bat │ ├── pipeline │ │ ├── build.gradle │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── airflow │ │ │ └── gcloud │ │ │ └── pipeline │ │ │ ├── Bq2BqPipeline.java │ │ │ └── CopyPipeline.java │ ├── settings.gradle │ └── transform │ │ ├── build.gradle │ │ └── src │ │ └── main │ │ └── java │ │ └── airflow │ │ └── gcloud │ │ ├── BqSchemaFor.java │ │ ├── data │ │ └── convert │ │ │ └── ConvertObjectToStringFn.java │ │ ├── model │ │ └── EmailEvent.java │ │ └── transform │ │ └── NoopFn.java └── afgc-spark-scala │ └── src │ └── main │ └── scala │ └── luigi │ └── gcloud │ └── spark │ └── Copy.scala └── img ├── airflow_connection.png ├── airflow_variables.png ├── console_service_account.png └── create_service_account.png /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/README.md -------------------------------------------------------------------------------- /dags/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dags/bigquery.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/dags/bigquery.py -------------------------------------------------------------------------------- /dags/cloud_storage.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/dags/cloud_storage.py -------------------------------------------------------------------------------- /dags/dataproc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/dags/dataproc.py -------------------------------------------------------------------------------- /dags/gcp_smoke/gsob_extract_day.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/dags/gcp_smoke/gsob_extract_day.sql -------------------------------------------------------------------------------- /dags/support/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dags/support/schemas.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/dags/support/schemas.py -------------------------------------------------------------------------------- /external/afgc-dataflow-java/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/external/afgc-dataflow-java/.gitignore -------------------------------------------------------------------------------- /external/afgc-dataflow-java/build.gradle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/external/afgc-dataflow-java/build.gradle -------------------------------------------------------------------------------- /external/afgc-dataflow-java/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/external/afgc-dataflow-java/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /external/afgc-dataflow-java/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/external/afgc-dataflow-java/gradle/wrapper/gradle-wrapper.properties -------------------------------------------------------------------------------- /external/afgc-dataflow-java/gradlew: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/external/afgc-dataflow-java/gradlew -------------------------------------------------------------------------------- /external/afgc-dataflow-java/gradlew.bat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/external/afgc-dataflow-java/gradlew.bat -------------------------------------------------------------------------------- /external/afgc-dataflow-java/pipeline/build.gradle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/external/afgc-dataflow-java/pipeline/build.gradle -------------------------------------------------------------------------------- /external/afgc-dataflow-java/pipeline/src/main/java/airflow/gcloud/pipeline/Bq2BqPipeline.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/external/afgc-dataflow-java/pipeline/src/main/java/airflow/gcloud/pipeline/Bq2BqPipeline.java -------------------------------------------------------------------------------- /external/afgc-dataflow-java/pipeline/src/main/java/airflow/gcloud/pipeline/CopyPipeline.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/external/afgc-dataflow-java/pipeline/src/main/java/airflow/gcloud/pipeline/CopyPipeline.java -------------------------------------------------------------------------------- /external/afgc-dataflow-java/settings.gradle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/external/afgc-dataflow-java/settings.gradle -------------------------------------------------------------------------------- /external/afgc-dataflow-java/transform/build.gradle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/external/afgc-dataflow-java/transform/build.gradle -------------------------------------------------------------------------------- /external/afgc-dataflow-java/transform/src/main/java/airflow/gcloud/BqSchemaFor.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/external/afgc-dataflow-java/transform/src/main/java/airflow/gcloud/BqSchemaFor.java -------------------------------------------------------------------------------- /external/afgc-dataflow-java/transform/src/main/java/airflow/gcloud/data/convert/ConvertObjectToStringFn.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/external/afgc-dataflow-java/transform/src/main/java/airflow/gcloud/data/convert/ConvertObjectToStringFn.java -------------------------------------------------------------------------------- /external/afgc-dataflow-java/transform/src/main/java/airflow/gcloud/model/EmailEvent.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/external/afgc-dataflow-java/transform/src/main/java/airflow/gcloud/model/EmailEvent.java -------------------------------------------------------------------------------- /external/afgc-dataflow-java/transform/src/main/java/airflow/gcloud/transform/NoopFn.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/external/afgc-dataflow-java/transform/src/main/java/airflow/gcloud/transform/NoopFn.java -------------------------------------------------------------------------------- /external/afgc-spark-scala/src/main/scala/luigi/gcloud/spark/Copy.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/external/afgc-spark-scala/src/main/scala/luigi/gcloud/spark/Copy.scala -------------------------------------------------------------------------------- /img/airflow_connection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/img/airflow_connection.png -------------------------------------------------------------------------------- /img/airflow_variables.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/img/airflow_variables.png -------------------------------------------------------------------------------- /img/console_service_account.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/img/console_service_account.png -------------------------------------------------------------------------------- /img/create_service_account.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexvanboxel/airflow-gcp-examples/HEAD/img/create_service_account.png --------------------------------------------------------------------------------