├── docs └── static │ ├── itp.png │ ├── barnacle.png │ ├── favicon.ico │ ├── StarThinker.png │ ├── deal_finder.png │ ├── itp_audit.png │ ├── policebot.png │ ├── transparency.png │ ├── StarThinker_All.png │ ├── StarThinker_Blue.png │ ├── StarThinker_Logo.png │ ├── StarThinker_Red.png │ ├── recipe_download.png │ ├── MonthlyBudgetMover.png │ ├── StarThinker_Green.png │ ├── StarThinker_White.png │ ├── StarThinker_Yellow.png │ ├── anonymize_dataset.png │ ├── floodlight_monitor.png │ ├── gTech_StarThinker.png │ ├── screenshot_client.png │ ├── screenshot_recipe.png │ ├── starthinker_ui_log.png │ ├── twitter_targeting.png │ ├── ctv_audience_affinity.png │ ├── itp_audit_dashboard.png │ ├── starthinker_ui_code.png │ ├── starthinker_ui_deploy.png │ ├── starthinker_ui_recipe.png │ ├── gTech_StarThinker_Large.png │ ├── screenshot_client_edit.png │ ├── screenshot_recipe_edit.png │ ├── starthinker_ui_project.png │ ├── starthinker_ui_solution.png │ ├── fonts │ └── roboto │ │ ├── Roboto-Bold.woff │ │ ├── Roboto-Bold.woff2 │ │ ├── Roboto-Light.woff │ │ ├── Roboto-Thin.woff │ │ ├── Roboto-Thin.woff2 │ │ ├── Roboto-Light.woff2 │ │ ├── Roboto-Medium.woff │ │ ├── Roboto-Medium.woff2 │ │ ├── Roboto-Regular.woff │ │ └── Roboto-Regular.woff2 │ └── starthinker_ui_solutions.png ├── tutorials ├── images │ ├── itp.png │ ├── deploy.png │ ├── splash.png │ ├── email_cm.png │ ├── gallery.png │ ├── recipes.png │ ├── barnacle_dv.png │ ├── bulk_dv360.png │ ├── bulkdozer.png │ ├── email_dv360.png │ ├── kv_uploader.png │ ├── logging_ui.png │ ├── ui_download.png │ ├── ui_manual.png │ ├── ui_project.png │ ├── ui_recipe.png │ ├── ui_solution.png │ ├── architecture.png │ ├── cloud_project.png │ ├── cloud_service.png │ ├── cm_user_edit.png │ ├── logging_logs.png │ ├── ui_recipe_map.png │ ├── ui_say_hello.png │ ├── verification.png │ ├── cloud_client_web.png │ ├── deploy_developer.png │ ├── deploy_project.png │ ├── deploy_service.png │ ├── dv360_targeter.png │ ├── recipe_download.png │ ├── segmentology_dv.png │ ├── starthinker_map.png │ ├── ui_recipe_create.png │ ├── audience_analysis.png │ ├── logging_dashboard.png │ ├── app_script_settings.png │ ├── campaign_comparison.png │ ├── dv360_targeting_audit.png │ ├── StarThinker_Whitepaper.pdf │ ├── cloud_client_installed.png │ └── dv360_feature_adoption.png ├── Bulkdozer │ └── Images │ │ ├── Bulkdozer_Reset.png │ │ ├── bulkdozer_log_tab.png │ │ ├── bulkdozer_ad_row_end.png │ │ ├── Bulkdozer_Campaign_tab.png │ │ ├── bulkdozer_ad_row_start.png │ │ ├── bulkdozer_campaign_row.png │ │ ├── bulkdozer_creative_row.png │ │ ├── bulkdozer_dropdown_menu.png │ │ ├── bulkdozer_ui_gtech_menu.png │ │ ├── bulkdozer_Landing_Page_row.png │ │ ├── bulkdozer_push_to_CM_menu.png │ │ ├── bulkdozer_ui_expanded_menu.png │ │ ├── bulkdozer_ui_recipe_name.png │ │ ├── bulkdozer_ui_recipe_plus.png │ │ ├── buulkdozer_ui_recipe_run.png │ │ ├── bulkdozer_Creative_Asset_Row.png │ │ ├── bulkdozer_load_entities_menu.png │ │ ├── bulkdozer_placement_row_end.png │ │ ├── bulkdozer_ui_recipe_schedule.png │ │ ├── bulkdozer_CM_created_campaign.png │ │ ├── bulkdozer_placement_row_start.png │ │ ├── bulkdozer_ui_recipe_accountId.png │ │ └── bulkdozer_ad_creative_assignment_row.png ├── cloud_project.md ├── cloud_client_installed.md ├── cloud_service.md ├── appscript_troubleshooting.md ├── data_schemas.md ├── cloud_client_web.md └── testing_ui.md ├── .dockerignore ├── starthinker_ui ├── static │ ├── itp.png │ ├── barnacle.png │ ├── favicon.ico │ ├── StarThinker.png │ ├── deal_finder.png │ ├── itp_audit.png │ ├── policebot.png │ ├── transparency.png │ ├── StarThinker_All.png │ ├── StarThinker_Blue.png │ ├── StarThinker_Logo.png │ ├── StarThinker_Red.png │ ├── recipe_download.png │ ├── MonthlyBudgetMover.png │ ├── StarThinker_Green.png │ ├── StarThinker_White.png │ ├── StarThinker_Yellow.png │ ├── anonymize_dataset.png │ ├── floodlight_monitor.png │ ├── gTech_StarThinker.png │ ├── screenshot_client.png │ ├── screenshot_recipe.png │ ├── starthinker_ui_log.png │ ├── twitter_targeting.png │ ├── ctv_audience_affinity.png │ ├── itp_audit_dashboard.png │ ├── starthinker_ui_code.png │ ├── starthinker_ui_deploy.png │ ├── starthinker_ui_recipe.png │ ├── gTech_StarThinker_Large.png │ ├── screenshot_client_edit.png │ ├── screenshot_recipe_edit.png │ ├── starthinker_ui_project.png │ ├── starthinker_ui_solution.png │ ├── fonts │ │ └── roboto │ │ │ ├── Roboto-Bold.woff │ │ │ ├── Roboto-Bold.woff2 │ │ │ ├── Roboto-Light.woff │ │ │ ├── Roboto-Thin.woff │ │ │ ├── Roboto-Thin.woff2 │ │ │ ├── Roboto-Light.woff2 │ │ │ ├── Roboto-Medium.woff │ │ │ ├── Roboto-Medium.woff2 │ │ │ ├── Roboto-Regular.woff │ │ │ └── Roboto-Regular.woff2 │ └── starthinker_ui_solutions.png ├── templates │ ├── errors │ │ └── 500.html │ ├── footer.html │ ├── form_setup.html │ └── input_checkbox_multiple.html ├── __init__.py ├── ui │ ├── __init__.py │ ├── wsgi.py │ └── urls.py ├── account │ ├── __init__.py │ ├── management │ │ ├── __init__.py │ │ └── commands │ │ │ └── __init__.py │ ├── migrations │ │ └── __init__.py │ ├── urls.py │ └── decorators.py ├── project │ ├── __init__.py │ ├── migrations │ │ └── __init__.py │ └── urls.py ├── recipe │ ├── __init__.py │ ├── management │ │ ├── __init__.py │ │ └── commands │ │ │ ├── __init__.py │ │ │ ├── recipe_usage.py │ │ │ └── autoscale.py │ ├── migrations │ │ └── __init__.py │ └── models_fields.py ├── website │ ├── __init__.py │ ├── management │ │ ├── __init__.py │ │ └── commands │ │ │ ├── __init__.py │ │ │ ├── airflow.py │ │ │ └── example.py │ ├── migrations │ │ └── __init__.py │ ├── templatetags │ │ └── __init__.py │ ├── models.py │ ├── urls.py │ ├── tests.py │ └── views.py └── manage.py ├── cron.yaml ├── .gcloudignore ├── tests ├── bqflow.json ├── archive.json ├── dataset.json ├── cm360_data_warehouse.json ├── dv360_data_warehouse.json ├── floodlight_monitor.json ├── twitter.json ├── dcm_log.json ├── fred.json ├── smartsheet.json ├── cm360_oculi.json ├── salesforce.json └── ctv_audience_affinity.json ├── .gitignore ├── AUTHORS ├── scripts ├── dynamite.json ├── say_hello.json ├── manual.json ├── sheets_clear.json ├── iam.json ├── dcm.json ├── drive_copy.json ├── bigquery_run_query.json ├── dcm_run.json ├── sheets_copy.json ├── dbm.json ├── archive.json ├── bucket.json ├── bigquery_function.json ├── dataset.json ├── bigquery_view.json ├── bigquery_query.json ├── test.json ├── hello.json ├── dcm_to_sheets.json ├── dbm_to_sheets.json ├── dcm_to_storage.json ├── dbm_to_storage.json ├── plx_to_bigquery.json ├── bigquery_storage.json ├── trends_places_to_bigquery_via_value.json ├── cm_campaign_audit.json ├── sheets_to_bigquery.json ├── kv_uploader.json └── interaction_analyzer.json ├── dags └── __init__.py ├── starthinker ├── airflow │ ├── __init__.py │ └── operators │ │ ├── __init__.py │ │ └── hello.py ├── task │ ├── __init__.py │ ├── dt │ │ ├── __init__.py │ │ └── schema │ │ │ └── __init__.py │ ├── ga │ │ ├── __init__.py │ │ └── run.py │ ├── sa │ │ ├── __init__.py │ │ └── run.py │ ├── archive │ │ └── __init__.py │ ├── bqflow │ │ └── __init__.py │ ├── bucket │ │ ├── __init__.py │ │ └── run.py │ ├── census │ │ └── __init__.py │ ├── dataset │ │ └── __init__.py │ ├── dbm │ │ └── __init__.py │ ├── dcm │ │ └── __init__.py │ ├── dcm_log │ │ └── __init__.py │ ├── drive │ │ ├── __init__.py │ │ └── run.py │ ├── email │ │ └── __init__.py │ ├── fred │ │ └── __init__.py │ ├── hello │ │ └── __init__.py │ ├── iam │ │ └── __init__.py │ ├── sdf │ │ ├── __init__.py │ │ └── run.py │ ├── sheets │ │ └── __init__.py │ ├── test │ │ └── __init__.py │ ├── traffic │ │ ├── __init__.py │ │ └── check.py │ ├── twitter │ │ └── __init__.py │ ├── url │ │ └── __init__.py │ ├── video │ │ └── __init__.py │ ├── anonymize │ │ └── __init__.py │ ├── barnacle │ │ └── __init__.py │ ├── bigquery │ │ └── __init__.py │ ├── cm_to_dv │ │ └── __init__.py │ ├── dv_editor │ │ └── __init__.py │ ├── dv_targeter │ │ └── __init__.py │ ├── google_api │ │ └── __init__.py │ ├── itp_audit │ │ └── __init__.py │ ├── salesforce │ │ ├── __init__.py │ │ └── run.py │ ├── smartsheet │ │ └── __init__.py │ ├── vision_api │ │ └── __init__.py │ ├── weather_gov │ │ ├── __init__.py │ │ └── check.py │ ├── conversion_upload │ │ └── __init__.py │ ├── dynamic_costs │ │ └── __init__.py │ ├── cm_report_replicate │ │ └── __init__.py │ ├── floodlight_monitor │ │ └── __init__.py │ └── monthly_budget_mover │ │ └── __init__.py ├── tool │ └── __init__.py ├── __init__.py └── util │ ├── dynamite.py │ └── salesforce.py ├── cloud_function └── requirements.txt ├── starthinker_assets └── README.md └── requirements.txt /docs/static/itp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/itp.png -------------------------------------------------------------------------------- /docs/static/barnacle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/barnacle.png -------------------------------------------------------------------------------- /docs/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/favicon.ico -------------------------------------------------------------------------------- /tutorials/images/itp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/itp.png -------------------------------------------------------------------------------- /docs/static/StarThinker.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/StarThinker.png -------------------------------------------------------------------------------- /docs/static/deal_finder.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/deal_finder.png -------------------------------------------------------------------------------- /docs/static/itp_audit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/itp_audit.png -------------------------------------------------------------------------------- /docs/static/policebot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/policebot.png -------------------------------------------------------------------------------- /tutorials/images/deploy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/deploy.png -------------------------------------------------------------------------------- /tutorials/images/splash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/splash.png -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.pyc 3 | *.swp 4 | .dockerignore 5 | starthinker_assets/ 6 | starthinker/legacy/ 7 | -------------------------------------------------------------------------------- /docs/static/transparency.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/transparency.png -------------------------------------------------------------------------------- /starthinker_ui/static/itp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/itp.png -------------------------------------------------------------------------------- /tutorials/images/email_cm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/email_cm.png -------------------------------------------------------------------------------- /tutorials/images/gallery.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/gallery.png -------------------------------------------------------------------------------- /tutorials/images/recipes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/recipes.png -------------------------------------------------------------------------------- /docs/static/StarThinker_All.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/StarThinker_All.png -------------------------------------------------------------------------------- /docs/static/StarThinker_Blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/StarThinker_Blue.png -------------------------------------------------------------------------------- /docs/static/StarThinker_Logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/StarThinker_Logo.png -------------------------------------------------------------------------------- /docs/static/StarThinker_Red.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/StarThinker_Red.png -------------------------------------------------------------------------------- /docs/static/recipe_download.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/recipe_download.png -------------------------------------------------------------------------------- /tutorials/images/barnacle_dv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/barnacle_dv.png -------------------------------------------------------------------------------- /tutorials/images/bulk_dv360.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/bulk_dv360.png -------------------------------------------------------------------------------- /tutorials/images/bulkdozer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/bulkdozer.png -------------------------------------------------------------------------------- /tutorials/images/email_dv360.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/email_dv360.png -------------------------------------------------------------------------------- /tutorials/images/kv_uploader.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/kv_uploader.png -------------------------------------------------------------------------------- /tutorials/images/logging_ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/logging_ui.png -------------------------------------------------------------------------------- /tutorials/images/ui_download.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/ui_download.png -------------------------------------------------------------------------------- /tutorials/images/ui_manual.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/ui_manual.png -------------------------------------------------------------------------------- /tutorials/images/ui_project.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/ui_project.png -------------------------------------------------------------------------------- /tutorials/images/ui_recipe.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/ui_recipe.png -------------------------------------------------------------------------------- /tutorials/images/ui_solution.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/ui_solution.png -------------------------------------------------------------------------------- /docs/static/MonthlyBudgetMover.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/MonthlyBudgetMover.png -------------------------------------------------------------------------------- /docs/static/StarThinker_Green.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/StarThinker_Green.png -------------------------------------------------------------------------------- /docs/static/StarThinker_White.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/StarThinker_White.png -------------------------------------------------------------------------------- /docs/static/StarThinker_Yellow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/StarThinker_Yellow.png -------------------------------------------------------------------------------- /docs/static/anonymize_dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/anonymize_dataset.png -------------------------------------------------------------------------------- /docs/static/floodlight_monitor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/floodlight_monitor.png -------------------------------------------------------------------------------- /docs/static/gTech_StarThinker.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/gTech_StarThinker.png -------------------------------------------------------------------------------- /docs/static/screenshot_client.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/screenshot_client.png -------------------------------------------------------------------------------- /docs/static/screenshot_recipe.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/screenshot_recipe.png -------------------------------------------------------------------------------- /docs/static/starthinker_ui_log.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/starthinker_ui_log.png -------------------------------------------------------------------------------- /docs/static/twitter_targeting.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/twitter_targeting.png -------------------------------------------------------------------------------- /starthinker_ui/static/barnacle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/barnacle.png -------------------------------------------------------------------------------- /starthinker_ui/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/favicon.ico -------------------------------------------------------------------------------- /tutorials/images/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/architecture.png -------------------------------------------------------------------------------- /tutorials/images/cloud_project.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/cloud_project.png -------------------------------------------------------------------------------- /tutorials/images/cloud_service.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/cloud_service.png -------------------------------------------------------------------------------- /tutorials/images/cm_user_edit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/cm_user_edit.png -------------------------------------------------------------------------------- /tutorials/images/logging_logs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/logging_logs.png -------------------------------------------------------------------------------- /tutorials/images/ui_recipe_map.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/ui_recipe_map.png -------------------------------------------------------------------------------- /tutorials/images/ui_say_hello.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/ui_say_hello.png -------------------------------------------------------------------------------- /tutorials/images/verification.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/verification.png -------------------------------------------------------------------------------- /cron.yaml: -------------------------------------------------------------------------------- 1 | cron: 2 | - description: "Worker Scale Protection" 3 | url: /recipe/autoscale/ 4 | schedule: every 3 minutes 5 | -------------------------------------------------------------------------------- /docs/static/ctv_audience_affinity.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/ctv_audience_affinity.png -------------------------------------------------------------------------------- /docs/static/itp_audit_dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/itp_audit_dashboard.png -------------------------------------------------------------------------------- /docs/static/starthinker_ui_code.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/starthinker_ui_code.png -------------------------------------------------------------------------------- /docs/static/starthinker_ui_deploy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/starthinker_ui_deploy.png -------------------------------------------------------------------------------- /docs/static/starthinker_ui_recipe.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/starthinker_ui_recipe.png -------------------------------------------------------------------------------- /starthinker_ui/static/StarThinker.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/StarThinker.png -------------------------------------------------------------------------------- /starthinker_ui/static/deal_finder.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/deal_finder.png -------------------------------------------------------------------------------- /starthinker_ui/static/itp_audit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/itp_audit.png -------------------------------------------------------------------------------- /starthinker_ui/static/policebot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/policebot.png -------------------------------------------------------------------------------- /tutorials/images/cloud_client_web.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/cloud_client_web.png -------------------------------------------------------------------------------- /tutorials/images/deploy_developer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/deploy_developer.png -------------------------------------------------------------------------------- /tutorials/images/deploy_project.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/deploy_project.png -------------------------------------------------------------------------------- /tutorials/images/deploy_service.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/deploy_service.png -------------------------------------------------------------------------------- /tutorials/images/dv360_targeter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/dv360_targeter.png -------------------------------------------------------------------------------- /tutorials/images/recipe_download.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/recipe_download.png -------------------------------------------------------------------------------- /tutorials/images/segmentology_dv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/segmentology_dv.png -------------------------------------------------------------------------------- /tutorials/images/starthinker_map.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/starthinker_map.png -------------------------------------------------------------------------------- /tutorials/images/ui_recipe_create.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/ui_recipe_create.png -------------------------------------------------------------------------------- /docs/static/gTech_StarThinker_Large.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/gTech_StarThinker_Large.png -------------------------------------------------------------------------------- /docs/static/screenshot_client_edit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/screenshot_client_edit.png -------------------------------------------------------------------------------- /docs/static/screenshot_recipe_edit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/screenshot_recipe_edit.png -------------------------------------------------------------------------------- /docs/static/starthinker_ui_project.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/starthinker_ui_project.png -------------------------------------------------------------------------------- /docs/static/starthinker_ui_solution.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/starthinker_ui_solution.png -------------------------------------------------------------------------------- /starthinker_ui/static/transparency.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/transparency.png -------------------------------------------------------------------------------- /tutorials/images/audience_analysis.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/audience_analysis.png -------------------------------------------------------------------------------- /tutorials/images/logging_dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/logging_dashboard.png -------------------------------------------------------------------------------- /docs/static/fonts/roboto/Roboto-Bold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/fonts/roboto/Roboto-Bold.woff -------------------------------------------------------------------------------- /docs/static/fonts/roboto/Roboto-Bold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/fonts/roboto/Roboto-Bold.woff2 -------------------------------------------------------------------------------- /docs/static/fonts/roboto/Roboto-Light.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/fonts/roboto/Roboto-Light.woff -------------------------------------------------------------------------------- /docs/static/fonts/roboto/Roboto-Thin.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/fonts/roboto/Roboto-Thin.woff -------------------------------------------------------------------------------- /docs/static/fonts/roboto/Roboto-Thin.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/fonts/roboto/Roboto-Thin.woff2 -------------------------------------------------------------------------------- /docs/static/starthinker_ui_solutions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/starthinker_ui_solutions.png -------------------------------------------------------------------------------- /starthinker_ui/static/StarThinker_All.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/StarThinker_All.png -------------------------------------------------------------------------------- /starthinker_ui/static/StarThinker_Blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/StarThinker_Blue.png -------------------------------------------------------------------------------- /starthinker_ui/static/StarThinker_Logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/StarThinker_Logo.png -------------------------------------------------------------------------------- /starthinker_ui/static/StarThinker_Red.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/StarThinker_Red.png -------------------------------------------------------------------------------- /starthinker_ui/static/recipe_download.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/recipe_download.png -------------------------------------------------------------------------------- /tutorials/images/app_script_settings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/app_script_settings.png -------------------------------------------------------------------------------- /tutorials/images/campaign_comparison.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/campaign_comparison.png -------------------------------------------------------------------------------- /tutorials/images/dv360_targeting_audit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/dv360_targeting_audit.png -------------------------------------------------------------------------------- /docs/static/fonts/roboto/Roboto-Light.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/fonts/roboto/Roboto-Light.woff2 -------------------------------------------------------------------------------- /docs/static/fonts/roboto/Roboto-Medium.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/fonts/roboto/Roboto-Medium.woff -------------------------------------------------------------------------------- /docs/static/fonts/roboto/Roboto-Medium.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/fonts/roboto/Roboto-Medium.woff2 -------------------------------------------------------------------------------- /docs/static/fonts/roboto/Roboto-Regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/fonts/roboto/Roboto-Regular.woff -------------------------------------------------------------------------------- /starthinker_ui/static/MonthlyBudgetMover.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/MonthlyBudgetMover.png -------------------------------------------------------------------------------- /starthinker_ui/static/StarThinker_Green.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/StarThinker_Green.png -------------------------------------------------------------------------------- /starthinker_ui/static/StarThinker_White.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/StarThinker_White.png -------------------------------------------------------------------------------- /starthinker_ui/static/StarThinker_Yellow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/StarThinker_Yellow.png -------------------------------------------------------------------------------- /starthinker_ui/static/anonymize_dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/anonymize_dataset.png -------------------------------------------------------------------------------- /starthinker_ui/static/floodlight_monitor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/floodlight_monitor.png -------------------------------------------------------------------------------- /starthinker_ui/static/gTech_StarThinker.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/gTech_StarThinker.png -------------------------------------------------------------------------------- /starthinker_ui/static/screenshot_client.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/screenshot_client.png -------------------------------------------------------------------------------- /starthinker_ui/static/screenshot_recipe.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/screenshot_recipe.png -------------------------------------------------------------------------------- /starthinker_ui/static/starthinker_ui_log.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/starthinker_ui_log.png -------------------------------------------------------------------------------- /starthinker_ui/static/twitter_targeting.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/twitter_targeting.png -------------------------------------------------------------------------------- /tutorials/images/StarThinker_Whitepaper.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/StarThinker_Whitepaper.pdf -------------------------------------------------------------------------------- /tutorials/images/cloud_client_installed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/cloud_client_installed.png -------------------------------------------------------------------------------- /tutorials/images/dv360_feature_adoption.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/images/dv360_feature_adoption.png -------------------------------------------------------------------------------- /docs/static/fonts/roboto/Roboto-Regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/docs/static/fonts/roboto/Roboto-Regular.woff2 -------------------------------------------------------------------------------- /starthinker_ui/static/ctv_audience_affinity.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/ctv_audience_affinity.png -------------------------------------------------------------------------------- /starthinker_ui/static/itp_audit_dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/itp_audit_dashboard.png -------------------------------------------------------------------------------- /starthinker_ui/static/starthinker_ui_code.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/starthinker_ui_code.png -------------------------------------------------------------------------------- /starthinker_ui/static/starthinker_ui_deploy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/starthinker_ui_deploy.png -------------------------------------------------------------------------------- /starthinker_ui/static/starthinker_ui_recipe.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/starthinker_ui_recipe.png -------------------------------------------------------------------------------- /tutorials/Bulkdozer/Images/Bulkdozer_Reset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/Bulkdozer/Images/Bulkdozer_Reset.png -------------------------------------------------------------------------------- /starthinker_ui/static/gTech_StarThinker_Large.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/gTech_StarThinker_Large.png -------------------------------------------------------------------------------- /starthinker_ui/static/screenshot_client_edit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/screenshot_client_edit.png -------------------------------------------------------------------------------- /starthinker_ui/static/screenshot_recipe_edit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/screenshot_recipe_edit.png -------------------------------------------------------------------------------- /starthinker_ui/static/starthinker_ui_project.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/starthinker_ui_project.png -------------------------------------------------------------------------------- /starthinker_ui/static/starthinker_ui_solution.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/starthinker_ui_solution.png -------------------------------------------------------------------------------- /tutorials/Bulkdozer/Images/bulkdozer_log_tab.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/Bulkdozer/Images/bulkdozer_log_tab.png -------------------------------------------------------------------------------- /starthinker_ui/static/fonts/roboto/Roboto-Bold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/fonts/roboto/Roboto-Bold.woff -------------------------------------------------------------------------------- /starthinker_ui/static/fonts/roboto/Roboto-Bold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/fonts/roboto/Roboto-Bold.woff2 -------------------------------------------------------------------------------- /starthinker_ui/static/fonts/roboto/Roboto-Light.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/fonts/roboto/Roboto-Light.woff -------------------------------------------------------------------------------- /starthinker_ui/static/fonts/roboto/Roboto-Thin.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/fonts/roboto/Roboto-Thin.woff -------------------------------------------------------------------------------- /starthinker_ui/static/fonts/roboto/Roboto-Thin.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/fonts/roboto/Roboto-Thin.woff2 -------------------------------------------------------------------------------- /starthinker_ui/static/starthinker_ui_solutions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/starthinker_ui_solutions.png -------------------------------------------------------------------------------- /tutorials/Bulkdozer/Images/bulkdozer_ad_row_end.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/Bulkdozer/Images/bulkdozer_ad_row_end.png -------------------------------------------------------------------------------- /starthinker_ui/static/fonts/roboto/Roboto-Light.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/fonts/roboto/Roboto-Light.woff2 -------------------------------------------------------------------------------- /starthinker_ui/static/fonts/roboto/Roboto-Medium.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/fonts/roboto/Roboto-Medium.woff -------------------------------------------------------------------------------- /starthinker_ui/static/fonts/roboto/Roboto-Medium.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/fonts/roboto/Roboto-Medium.woff2 -------------------------------------------------------------------------------- /starthinker_ui/static/fonts/roboto/Roboto-Regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/fonts/roboto/Roboto-Regular.woff -------------------------------------------------------------------------------- /tutorials/Bulkdozer/Images/Bulkdozer_Campaign_tab.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/Bulkdozer/Images/Bulkdozer_Campaign_tab.png -------------------------------------------------------------------------------- /tutorials/Bulkdozer/Images/bulkdozer_ad_row_start.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/Bulkdozer/Images/bulkdozer_ad_row_start.png -------------------------------------------------------------------------------- /tutorials/Bulkdozer/Images/bulkdozer_campaign_row.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/Bulkdozer/Images/bulkdozer_campaign_row.png -------------------------------------------------------------------------------- /tutorials/Bulkdozer/Images/bulkdozer_creative_row.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/Bulkdozer/Images/bulkdozer_creative_row.png -------------------------------------------------------------------------------- /tutorials/Bulkdozer/Images/bulkdozer_dropdown_menu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/Bulkdozer/Images/bulkdozer_dropdown_menu.png -------------------------------------------------------------------------------- /tutorials/Bulkdozer/Images/bulkdozer_ui_gtech_menu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/Bulkdozer/Images/bulkdozer_ui_gtech_menu.png -------------------------------------------------------------------------------- /starthinker_ui/static/fonts/roboto/Roboto-Regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/starthinker_ui/static/fonts/roboto/Roboto-Regular.woff2 -------------------------------------------------------------------------------- /tutorials/Bulkdozer/Images/bulkdozer_Landing_Page_row.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/Bulkdozer/Images/bulkdozer_Landing_Page_row.png -------------------------------------------------------------------------------- /tutorials/Bulkdozer/Images/bulkdozer_push_to_CM_menu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/Bulkdozer/Images/bulkdozer_push_to_CM_menu.png -------------------------------------------------------------------------------- /tutorials/Bulkdozer/Images/bulkdozer_ui_expanded_menu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/Bulkdozer/Images/bulkdozer_ui_expanded_menu.png -------------------------------------------------------------------------------- /tutorials/Bulkdozer/Images/bulkdozer_ui_recipe_name.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/Bulkdozer/Images/bulkdozer_ui_recipe_name.png -------------------------------------------------------------------------------- /tutorials/Bulkdozer/Images/bulkdozer_ui_recipe_plus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/Bulkdozer/Images/bulkdozer_ui_recipe_plus.png -------------------------------------------------------------------------------- /tutorials/Bulkdozer/Images/buulkdozer_ui_recipe_run.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/Bulkdozer/Images/buulkdozer_ui_recipe_run.png -------------------------------------------------------------------------------- /tutorials/Bulkdozer/Images/bulkdozer_Creative_Asset_Row.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/Bulkdozer/Images/bulkdozer_Creative_Asset_Row.png -------------------------------------------------------------------------------- /tutorials/Bulkdozer/Images/bulkdozer_load_entities_menu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/Bulkdozer/Images/bulkdozer_load_entities_menu.png -------------------------------------------------------------------------------- /tutorials/Bulkdozer/Images/bulkdozer_placement_row_end.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/Bulkdozer/Images/bulkdozer_placement_row_end.png -------------------------------------------------------------------------------- /tutorials/Bulkdozer/Images/bulkdozer_ui_recipe_schedule.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/Bulkdozer/Images/bulkdozer_ui_recipe_schedule.png -------------------------------------------------------------------------------- /tutorials/Bulkdozer/Images/bulkdozer_CM_created_campaign.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/Bulkdozer/Images/bulkdozer_CM_created_campaign.png -------------------------------------------------------------------------------- /tutorials/Bulkdozer/Images/bulkdozer_placement_row_start.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/Bulkdozer/Images/bulkdozer_placement_row_start.png -------------------------------------------------------------------------------- /tutorials/Bulkdozer/Images/bulkdozer_ui_recipe_accountId.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/Bulkdozer/Images/bulkdozer_ui_recipe_accountId.png -------------------------------------------------------------------------------- /tutorials/Bulkdozer/Images/bulkdozer_ad_creative_assignment_row.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/starthinker/HEAD/tutorials/Bulkdozer/Images/bulkdozer_ad_creative_assignment_row.png -------------------------------------------------------------------------------- /.gcloudignore: -------------------------------------------------------------------------------- 1 | .*~ 2 | .pyc 3 | .git 4 | .gitignore 5 | *__pycache__/ 6 | starthinker_cron/ 7 | starthinker_assets/ 8 | starthinker_airflow/ 9 | starthinker_database/ 10 | starthinker_virtualenv/ 11 | -------------------------------------------------------------------------------- /tests/bqflow.json: -------------------------------------------------------------------------------- 1 | { 2 | "setup":{ 3 | "license":"Licensed under the Apache License, Version 2.0", 4 | "copyright":"Copyright 2020 Google LLC" 5 | }, 6 | "tasks":[ 7 | { "bqflow":{ 8 | "auth":"service" 9 | }} 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.pyc 3 | *.swp 4 | *.yaml 5 | __pycache__ 6 | starthinker_cron/ 7 | starthinker_database/ 8 | starthinker_virtualenv/ 9 | starthinker_assets/*.json 10 | starthinker_assets/config.sh 11 | starthinker_ui/recipe/scripts_lookup.py 12 | starthinker_ui/*/migrations/0*.py 13 | tests/config.json 14 | tests/recipes/ 15 | tests/logs/ 16 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # This is the official list of StarThinker authors for copyright purposes. 2 | # This file is distinct from the CONTRIBUTORS files. 3 | # See the latter for an explanation. 4 | 5 | # Names should be added to this file as: 6 | # Name or Organization 7 | # The email address is not required for organizations. 8 | 9 | Paul Kenjora 10 | Mauricio Desiderio 11 | John Terwilleger 12 | -------------------------------------------------------------------------------- /tutorials/cloud_project.md: -------------------------------------------------------------------------------- 1 | # StarThinker Cloud Project ID 2 | 3 | ## Get Cloud Project ID 4 | 5 | Retrieve a Cloud Project ID from: https://console.cloud.google.com/ 6 | 7 | ![Cloud Project](images/cloud_project.png) 8 | [View Cloud Project ID](images/cloud_project.png) 9 | 10 | ### Retrieval Steps 11 | 12 | 1. Google Cloud Project -> Project Drop Down At The Top -> Column With ID 13 | - Be sure to copy the ID not the name. 14 | 15 | --- 16 | © 2019 Google Inc. - Apache License, Version 2.0 17 | -------------------------------------------------------------------------------- /tests/archive.json: -------------------------------------------------------------------------------- 1 | { 2 | "setup":{ 3 | "license":"Licensed under the Apache License, Version 2.0", 4 | "copyright":"Copyright 2020 Google LLC" 5 | }, 6 | "tasks":[ 7 | { "include":{ 8 | "script":"scripts/archive.json", 9 | "parameters":{ 10 | "archive_days":365, 11 | "archive_bucket":{"field":{ "name":"archive_bucket", "kind":"string", "description":"" }}, 12 | "archive_path":{"field":{ "name":"archive_path", "kind":"string", "description":"No leading slash and no asterix needed." }}, 13 | "archive_delete":false 14 | } 15 | }} 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /starthinker_ui/templates/errors/500.html: -------------------------------------------------------------------------------- 1 | 16 | 17 | Error 18 | -------------------------------------------------------------------------------- /tests/dataset.json: -------------------------------------------------------------------------------- 1 | { 2 | "setup":{ 3 | "license":"Licensed under the Apache License, Version 2.0", 4 | "copyright":"Copyright 2020 Google LLC" 5 | }, 6 | "tasks":[ 7 | { "dataset":{ 8 | "auth":"service", 9 | "dataset":{"field":{ "name":"test_run_id", "kind":"string", "description":"The dataset to use for the test.", "prefix":"StarThinker_Test_Dataset_", "default": "Manual"}}, 10 | "clear":true 11 | }}, 12 | { "dataset":{ 13 | "auth":"service", 14 | "dataset":{"field":{ "name":"test_run_id", "kind":"string", "description":"The dataset to use for the test.", "prefix":"StarThinker_Test_Dataset_", "default": "Manual"}}, 15 | "delete":true 16 | }} 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /scripts/dynamite.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2017-08-14", 4 | "icon":"chat", 5 | "title":"Dynamite", 6 | "description":"Write a notification to a dynamite room.", 7 | "authors":["kenjora@google.com"], 8 | "private":["kenjora@google.com"], 9 | "from":["python"], 10 | "to":["dynamite"] 11 | }, 12 | "tasks":[ 13 | { "dynamite":{ 14 | "room":{"field":{ "name":"dynamite_room", "kind":"string", "order":1, "default":"" }}, 15 | "key":{"field":{ "name":"dynamite_key", "kind":"string", "order":2, "default":"" }}, 16 | "token":{"field":{ "name":"dynamite_token", "kind":"string", "order":3, "default":"" }}, 17 | "message":{"field":{ "name":"dynamite_message", "kind":"string", "order":4, "default":"" }} 18 | }} 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /dags/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### -------------------------------------------------------------------------------- /starthinker_ui/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/airflow/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/dt/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/ga/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2021 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/sa/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2021 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/tool/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2021 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker_ui/ui/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/archive/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/bqflow/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/bucket/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/census/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/dataset/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/dbm/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/dcm/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/dcm_log/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/drive/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/email/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/fred/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/hello/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/iam/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/sdf/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/sheets/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/test/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/traffic/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/twitter/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/url/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2021 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/video/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker_ui/account/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker_ui/project/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker_ui/recipe/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker_ui/website/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/airflow/operators/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/anonymize/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/barnacle/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/bigquery/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/cm_to_dv/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/dt/schema/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/dv_editor/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/dv_targeter/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/google_api/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/itp_audit/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/salesforce/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/smartsheet/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/vision_api/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2021 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/weather_gov/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/conversion_upload/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/dynamic_costs/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker_ui/account/management/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker_ui/account/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker_ui/project/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker_ui/recipe/management/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker_ui/recipe/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker_ui/website/management/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker_ui/website/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/cm_report_replicate/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/floodlight_monitor/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker/task/monthly_budget_mover/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker_ui/website/templatetags/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /cloud_function/requirements.txt: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2021 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | 19 | starthinker 20 | -------------------------------------------------------------------------------- /starthinker_ui/account/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker_ui/recipe/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /starthinker_ui/website/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | -------------------------------------------------------------------------------- /scripts/say_hello.json: -------------------------------------------------------------------------------- 1 | { 2 | "setup":{ 3 | "license":"Licensed under the Apache License, Version 2.0", 4 | "copyright":"Copyright 2020 Google LLC", 5 | "ui":false, 6 | "id":"cloud-project-id", 7 | "week":["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"], 8 | "hour":[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23], 9 | "auth":{ 10 | "client":"/home/.credentials/project_client.json", 11 | "service":"/home/.credentials/project_service.json", 12 | "user":"/home/.credentials/project_user.json" 13 | } 14 | }, 15 | "tasks":[ 16 | { "hello":{ 17 | "auth":{"field":{ "name":"auth_read", "kind":"authentication", "order":1, "default":"user", "description":"Credentials used for reading data." }}, 18 | "say":"Hello World" 19 | } 20 | } 21 | ] 22 | } 23 | -------------------------------------------------------------------------------- /scripts/manual.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2019-07-17", 4 | "license":"Licensed under the Apache License, Version 2.0", 5 | "copyright":"Copyright 2020 Google LLC", 6 | "ui":false, 7 | "icon":"notifications", 8 | "title":"Test Script", 9 | "description":"Used by tests.", 10 | "instructions":[ 11 | "This should be called by the tests scripts only.", 12 | "When run will generate a say hello log." 13 | ], 14 | "authors":["kenjora@google.com"] 15 | }, 16 | "setup":{ 17 | "day":[], 18 | "hour":[] 19 | }, 20 | "tasks":[ 21 | { "hello":{ 22 | "auth":{"field":{ "name":"auth_read", "kind":"authentication", "order":1, "default":"user", "description":"Credentials used for reading data." }}, 23 | "hour":[], 24 | "say":"Hello Manual", 25 | "sleep":0 26 | }} 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /starthinker/__init__.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | 19 | VERSION = (0, 0, 4) 20 | 21 | __version__ = '.'.join(map(str, VERSION)) 22 | -------------------------------------------------------------------------------- /starthinker_ui/website/models.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | ########################################################################### 4 | # 5 | # Copyright 2020 Google LLC 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | ########################################################################### 20 | 21 | from django.db import models 22 | -------------------------------------------------------------------------------- /tests/cm360_data_warehouse.json: -------------------------------------------------------------------------------- 1 | { 2 | "setup":{ 3 | "license":"Licensed under the Apache License, Version 2.0", 4 | "copyright":"Copyright 2020 Google LLC" 5 | }, 6 | "tasks": [ 7 | { "dataset":{ 8 | "auth":"service", 9 | "dataset":{"field":{ "name":"test_run_id", "kind":"string", "description":"The dataset to use for the test.", "prefix":"StarThinker_Test_Warehouse_CM_", "default": "Manual"}}, 10 | "clear":true 11 | }}, 12 | { "include":{ 13 | "script":"scripts/cm360_data_warehouse.json", 14 | "parameters":{ 15 | "auth_bigquery":"service", 16 | "auth_cm":"user", 17 | "recipe_slug":{"field":{ "name":"test_run_id", "kind":"string", "description":"The dataset to use for the test.", "prefix":"StarThinker_Test_Warehouse_CM_", "default": "Manual"}}, 18 | "accounts":{"field":{ "name":"accounts", "kind":"integer_list", "description":"Comma separated DCM account ids." }} 19 | } 20 | }} 21 | ] 22 | } 23 | -------------------------------------------------------------------------------- /tests/dv360_data_warehouse.json: -------------------------------------------------------------------------------- 1 | { 2 | "setup":{ 3 | "license":"Licensed under the Apache License, Version 2.0", 4 | "copyright":"Copyright 2020 Google LLC" 5 | }, 6 | "tasks": [ 7 | { "dataset":{ 8 | "auth":"service", 9 | "dataset":{"field":{ "name":"test_run_id", "kind":"string", "description":"The dataset to use for the test.", "prefix":"StarThinker_Test_Warehouse_DV_", "default": "Manual"}}, 10 | "clear":false 11 | }}, 12 | { "include":{ 13 | "script":"scripts/dv360_data_warehouse.json", 14 | "parameters":{ 15 | "auth_bigquery":"service", 16 | "auth_dv":"user", 17 | "recipe_slug":{"field":{ "name":"test_run_id", "kind":"string", "description":"The dataset to use for the test.", "prefix":"StarThinker_Test_Warehouse_DV_", "default": "Manual"}}, 18 | "partners":{"field":{ "name":"partners", "kind":"integer_list", "description":"Comma separated DV partner ids." }} 19 | } 20 | }} 21 | ] 22 | } 23 | -------------------------------------------------------------------------------- /tests/floodlight_monitor.json: -------------------------------------------------------------------------------- 1 | { 2 | "setup":{ 3 | "license":"Licensed under the Apache License, Version 2.0", 4 | "copyright":"Copyright 2020 Google LLC" 5 | }, 6 | "tasks":[ 7 | { "sheets":{ 8 | "__comment__":"Copy the tamplate sheet to the users test sheet.", 9 | "auth":"user", 10 | "template":{ 11 | "sheet":"https://docs.google.com/spreadsheets/d/1fQnxfrBCieIifx9-NOX-4jeAqzHTBEvPIdOyaT3f9nw/", 12 | "tab":"Floodlight Monitor" 13 | }, 14 | "sheet":"StarThinker Test Floodlight Monitor", 15 | "tab":"Floodlight Monitor", 16 | "range":"A1:D" 17 | }}, 18 | { "include":{ 19 | "script":"scripts/floodlight_monitor.json", 20 | "parameters":{ 21 | "dcm_account":{"field":{ "name":"account_id", "kind":"string", "default":"" }}, 22 | "sheet":"StarThinker Test Floodlight Monitor" 23 | } 24 | }}, 25 | { 26 | "test" : { 27 | "assert":"Floodlight Monitor ran." 28 | } 29 | } 30 | ] 31 | } 32 | -------------------------------------------------------------------------------- /tests/twitter.json: -------------------------------------------------------------------------------- 1 | { 2 | "setup":{ 3 | "license":"Licensed under the Apache License, Version 2.0", 4 | "copyright":"Copyright 2020 Google LLC" 5 | }, 6 | "tasks":[ 7 | { "include":{ 8 | "script":"scripts/twitter.json", 9 | "parameters":{ 10 | "recipe_name":{"field":{ "name":"test_run_id", "kind":"string", "description":"The sheet to use for the test.", "prefix":"StarThinker Test Twitter ", "default": "Manual"}}, 11 | "recipe_slug":{"field":{ "name":"test_run_id", "kind":"string", "description":"The dataset to use for the test.", "prefix":"StarThinker_Test_Twitter_", "default": "Manual"}}, 12 | "recipe_project":{"field":{ "name":"recipe_project", "kind":"string", "description":"Google Cloud project id." }}, 13 | "twitter_secret":{"field":{ "name":"twitter_secret", "kind":"string", "description":"Twitter API secret token." }}, 14 | "twitter_key":{"field":{ "name":"twitter_key", "kind":"string", "description":"Twitter API key token." }} 15 | } 16 | }} 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /tutorials/cloud_client_installed.md: -------------------------------------------------------------------------------- 1 | # StarThinker Command Line Client Credentials 2 | 3 | ## Create Client Credentials ( Desktop ) 4 | 5 | Retrieve oAuth Client Id from: https://console.cloud.google.com/apis/credentials 6 | 7 | ![Cloud Client Other](images/cloud_client_installed.png) 8 | [View Cloud Client Other Setup Larger](images/cloud_client_installed.png) 9 | 10 | ### Setup Steps 11 | 12 | 1. Google Cloud Project -> APIs & Services -> Credentials -> Create Credentials -> oAuth Client Id. 13 | 1. Select Desktop. 14 | 1. Name is StarThinker. 15 | 1. Click the download icon next to the new credentials and open the downloaded file. 16 | 1. Copy the contents and paste into the StarThinker deployment script when prompted for Client Installed Credentials. 17 | 18 | ### Verify 19 | 20 | If all goes well, after deployment, the following files should exist with your credentials inside: 21 | 22 | - starthinker_assets/client_installed.json 23 | - starthinker_assets/user.json 24 | 25 | --- 26 | © 2019 Google Inc. - Apache License, Version 2.0 27 | -------------------------------------------------------------------------------- /starthinker_assets/README.md: -------------------------------------------------------------------------------- 1 | # StarThinker Assets 2 | 3 | This directory will hold all assets for a starthinker configuration. 4 | 5 | All assets are created using: 6 | 7 | ```source starthinker/install/deploy.sh``` 8 | 9 | Assets include... 10 | 11 | - production.sh - all paths and cloud project variables. 12 | - development.sh - all paths and cloud project variables. 13 | - client.json - client credentials. 14 | - service.josn - service credentials. 15 | - env - virtual environment configuration. 16 | 17 | If running developer deployment, load a StarThinker environment using: 18 | 19 | ``` 20 | source starthinker_assets/development.sh 21 | ``` 22 | 23 | For production, load a StarThinker environment using: 24 | 25 | ``` 26 | source starthinker_assets/development.sh 27 | ``` 28 | 29 | You are ready to start creating StarThinker recipes. 30 | 31 | In production to access the database try: 32 | 33 | psql -h localhost -d $STARTHINKER_UI_DATABSE_NAME -U $STARTHINKER_UI_DATABASE_USER 34 | 35 | --- 36 | © 2019 Google Inc. - Apache License, Version 2.0 37 | -------------------------------------------------------------------------------- /starthinker_ui/templates/footer.html: -------------------------------------------------------------------------------- 1 | {% comment %} 2 | ########################################################################### 3 | # 4 | # Copyright 2020 Google LLC 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | ########################################################################### 19 | {% endcomment %} 20 | 21 |
22 |
23 |
24 | © 2019 Google LLC 25 |
26 |
27 |
28 | -------------------------------------------------------------------------------- /tutorials/cloud_service.md: -------------------------------------------------------------------------------- 1 | # StarThinker Cloud Service Credentials 2 | 3 | ## Create Service Credentials 4 | 5 | Retrieve Service Account Key Credentials from: https://console.cloud.google.com/apis/credentials 6 | 7 | ![Cloud Service](images/cloud_service.png) 8 | [View Cloud Service Setup Larger](images/cloud_service.png) 9 | 10 | ### Setup Steps 11 | 12 | 1. Google Cloud Project -> APIs & Services -> Credentials -> Create Credentials -> Service Account Key. 13 | 1. New Service Account. 14 | 1. Role is Project Owner. 15 | 1. Service Account Name is starthinker. 16 | 1. Key Type is JSON. 17 | 1. Click Create and open the downloaded file. 18 | 1. Copy the contents and paste into the StarThinker deployment script when prompted for Service Credentials. 19 | - You may have to click CTRL+D after pasting the service crdentials. 20 | 21 | ### Verify 22 | 23 | If all goes well, after deployment, the following file should exist with your credentials inside: 24 | 25 | - starthinker_assets/service.json 26 | 27 | --- 28 | © 2019 Google Inc. - Apache License, Version 2.0 29 | -------------------------------------------------------------------------------- /scripts/sheets_clear.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2017-12-07", 4 | "license":"Licensed under the Apache License, Version 2.0", 5 | "copyright":"Copyright 2020 Google LLC", 6 | "icon":"view_column", 7 | "title":"Sheet Clear", 8 | "description":"Clear data from a sheet.", 9 | "test":"https://github.com/google/starthinker/blob/master/tests/sheets.json", 10 | "from":["sheets"], 11 | "to":["sheets"], 12 | "instructions":[ 13 | "For the sheet, provide the full edit URL." 14 | ], 15 | "authors":["mauriciod@google.com"] 16 | }, 17 | "tasks":[ 18 | { "sheets":{ 19 | "auth":{"field":{ "name":"auth_read", "kind":"authentication", "order":1, "default":"user", "description":"Credentials used for reading data." }}, 20 | "sheet":{"field":{ "name":"sheets_sheet", "kind":"string", "order":1, "default":"" }}, 21 | "tab":{"field":{ "name":"sheets_tab", "kind":"string", "order":2, "default":"" }}, 22 | "range":{"field":{ "name":"sheets_range", "kind":"string", "order":3, "default":"" }}, 23 | "clear":true 24 | }} 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /starthinker_ui/account/urls.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | 19 | from django.conf.urls import url 20 | 21 | from starthinker_ui.account import views 22 | 23 | urlpatterns = [ 24 | url(r'^oauth_callback/$', 25 | views.oauth_callback, 26 | name='account.oauth.callback'), 27 | url(r'^logout/$', views.logout, name='account.logout'), 28 | ] 29 | -------------------------------------------------------------------------------- /scripts/iam.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2018-10-02", 4 | "license":"Licensed under the Apache License, Version 2.0", 5 | "copyright":"Copyright 2020 Google LLC", 6 | "icon":"file_download", 7 | "title":"Project IAM", 8 | "description":"Sets project permissions for an email.", 9 | "from":["iam"], 10 | "to":["iam"], 11 | "instructions":[ 12 | "Provide a role in the form of projects/[project name]/roles/[role name]", 13 | "Enter an email to grant that role to.", 14 | "This only grants roles, you must remove them from the project manually." 15 | ], 16 | "authors":["kenjora@google.com"] 17 | }, 18 | "tasks":[ 19 | { "iam":{ 20 | "auth":{"field":{ "name":"auth_write", "kind":"authentication", "order":1, "default":"service", "description":"Credentials used for writing data." }}, 21 | "role":{"field":{ "name":"role", "kind":"string", "order":1, "default":"", "description":"projects/[project name]/roles/[role name]" }}, 22 | "email":{"field":{ "name":"email", "kind":"string", "order":2, "default":"", "description":"Email address to grant role to." }} 23 | }} 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /starthinker_ui/website/urls.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | 19 | from django.conf.urls import url 20 | 21 | from starthinker_ui.website import views 22 | 23 | urlpatterns = [ 24 | url(r'^help/$', views.help, name='website.help'), 25 | url(r'^solution/$', views.solutions, name='website.solutions'), 26 | url(r'^solution/(?P\w+)/$', views.solution, name='website.solution'), 27 | ] 28 | -------------------------------------------------------------------------------- /starthinker_ui/website/tests.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | 19 | from django.test import TestCase 20 | 21 | 22 | class WebsiteTest(TestCase): 23 | 24 | def test_ui_solution(self): 25 | resp = self.client.get('/solution/') 26 | self.assertEqual(resp.status_code, 200) 27 | 28 | def test_ui_help(self): 29 | resp = self.client.get('/help/') 30 | self.assertEqual(resp.status_code, 200) 31 | -------------------------------------------------------------------------------- /scripts/dcm.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2017-08-14", 4 | "license":"Licensed under the Apache License, Version 2.0", 5 | "copyright":"Copyright 2020 Google LLC", 6 | "icon":"donut_small", 7 | "from":["cm360"], 8 | "to":["cm360"], 9 | "title":"CM360 Report", 10 | "description":"Create a CM report from a JSON definition.", 11 | "instructions":[ 12 | "Add a an account as [account_id]@[profile_id]", 13 | "Fetch the report JSON definition. Arguably could be better.", 14 | "The account is automatically added to the report definition." 15 | ], 16 | "authors":["kenjora@google.com"] 17 | }, 18 | "tasks":[ 19 | { "dcm":{ 20 | "auth":{"field":{ "name":"auth_read", "kind":"authentication", "order":1, "default":"user", "description":"Credentials used for reading data." }}, 21 | "report":{ 22 | "account":{"field":{ "name":"account", "kind":"string", "order":1, "default":"" }}, 23 | "body":{"field":{ "name":"body", "kind":"json", "order":2, "default":"{}" }} 24 | }, 25 | "delete":{"field":{ "name":"delete", "kind":"boolean", "order":3, "default":false }} 26 | }} 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /tutorials/appscript_troubleshooting.md: -------------------------------------------------------------------------------- 1 | # App Script Troubleshooting 2 | 3 | ## Common AppScript Settings 4 | 5 | ![AppScript Troubleshooting](images/app_script_settings.png) 6 | 7 | --- 8 | ### Tool controls are not showing in the menu at the top of the sheet? 9 | 10 | For older scripts you need to **Disable new App Script runtime powered by Chrome V8**, navigate to 11 | **Script Editor** then from the **Run** menu ensure Chrome V8 runtime is disabled. When you close 12 | the script edit tab and refresh your sheet, the tools menu should appear at the top. 13 | 14 | --- 15 | ### An error indicating the API is not enabled appears? 16 | 17 | Google services need to be turned on for each script. Usually the script has the correct services active 18 | at the time it is built, but sometimes servies may need to be turned on or API versions changed. Navigate 19 | to **Script Editor** then from the **Resources** menu choose **Advanced Google Services**. From the 20 | dropdown ensure the API and version matches the endpoint you are connecting to. Changing versions may 21 | break the script and require a development to fix. 22 | 23 | --- 24 | © 2019 Google Inc. - Apache License, Version 2.0 25 | -------------------------------------------------------------------------------- /tests/dcm_log.json: -------------------------------------------------------------------------------- 1 | { 2 | "setup":{ 3 | "license":"Licensed under the Apache License, Version 2.0", 4 | "copyright":"Copyright 2020 Google LLC" 5 | }, 6 | "tasks":[ 7 | { "dataset":{ 8 | "auth":"service", 9 | "dataset":{"field":{ "name":"test_run_id", "kind":"string", "description":"The dataset to use for the test.", "prefix":"StarThinker_Test_CM_Log_", "default": "Manual"}}, 10 | "clear":true 11 | }}, 12 | { "include":{ 13 | "script":"scripts/dcm_log.json", 14 | "parameters":{ 15 | "auth_read":"user", 16 | "auth_write":"service", 17 | "accounts":{"field":{ "name":"accounts", "kind":"integer_list", "description":"Comma separated DCM account ids." }}, 18 | "recipe_slug":{"field":{ "name":"test_run_id", "kind":"string", "description":"The dataset to use for the test.", "prefix":"StarThinker_Test_CM_Log_", "default": "Manual"}} 19 | } 20 | }}, 21 | { "dataset":{ 22 | "auth":"service", 23 | "dataset":{"field":{ "name":"test_run_id", "kind":"string", "description":"The dataset to use for the test.", "prefix":"StarThinker_Test_CM_Log_", "default": "Manual"}}, 24 | "delete":true 25 | }} 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /tests/fred.json: -------------------------------------------------------------------------------- 1 | { 2 | "setup":{ 3 | "license":"Licensed under the Apache License, Version 2.0", 4 | "copyright":"Copyright 2020 Google LLC" 5 | }, 6 | "tasks":[ 7 | { "include":{ 8 | "script":"scripts/fred_series_to_bigquery.json", 9 | "parameters":{ 10 | "auth":"service", 11 | "fred_api_key":{"field":{ "name":"fred_api_key", "kind":"string", "description":"Fred API key." }}, 12 | "fred_series_id":"DAUPSA", 13 | "fred_units":"lin", 14 | "fred_frequency":"", 15 | "fred_aggregation_method":"sum", 16 | "project":"kenjora-161023", 17 | "dataset":"Test" 18 | } 19 | }}, 20 | { "include":{ 21 | "script":"scripts/fred_regional_to_bigquery.json", 22 | "parameters":{ 23 | "auth":"service", 24 | "fred_api_key":{"field":{ "name":"fred_api_key", "kind":"string", "description":"Fred API key." }}, 25 | "fred_series_group":"882", 26 | "fred_region_type":"county", 27 | "fred_units":"lin", 28 | "fred_frequency":"", 29 | "fred_aggregation_method":"sum", 30 | "project":"kenjora-161023", 31 | "dataset":"Test" 32 | } 33 | }} 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /scripts/drive_copy.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2019-08-28", 4 | "license":"Licensed under the Apache License, Version 2.0", 5 | "copyright":"Copyright 2020 Google LLC", 6 | "icon":"file_copy", 7 | "title":"Drive Copy", 8 | "description":"Copy a drive document.", 9 | "test":"https://github.com/google/starthinker/blob/master/tests/drive.json", 10 | "from":["drive"], 11 | "to":["drive"], 12 | "instructions":[ 13 | "Specify a source URL or document name.", 14 | "Specify a destination name.", 15 | "If destination does not exist, source will be copied." 16 | ], 17 | "authors":["kenjora@google.com"] 18 | }, 19 | "tasks":[ 20 | { "drive":{ 21 | "auth":{"field":{ "name":"auth_read", "kind":"authentication", "order":1, "default":"user", "description":"Credentials used for reading data." }}, 22 | "copy":{ 23 | "source":{"field":{ "name":"source", "kind":"string", "order":1, "default":"", "description":"Name or URL of document to copy from." }}, 24 | "destination":{"field":{ "name":"destination", "kind":"string", "order":2, "default":"", "description":"Name document to copy to." }} 25 | } 26 | }} 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /scripts/bigquery_run_query.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2017-08-14", 4 | "license":"Licensed under the Apache License, Version 2.0", 5 | "copyright":"Copyright 2020 Google LLC", 6 | "icon":"arrow_forward", 7 | "title":"BigQuery Query Run", 8 | "description":"Run query on a project.", 9 | "from":["bigquery"], 10 | "to":["bigquery"], 11 | "instructions":[ 12 | "Specify a single query and choose legacy or standard mode.", 13 | "For PLX use: SELECT * FROM [plx.google:FULL_TABLE_NAME.all] WHERE...", 14 | "For non legacy use: SELECT * `project.datset.table` WHERE..." 15 | ], 16 | "authors":["terwilleger@google.com"] 17 | }, 18 | "tasks":[ 19 | { "bigquery":{ 20 | "auth":{"field":{ "name":"auth_write", "kind":"authentication", "order":1, "default":"service", "description":"Credentials used for writing data." }}, 21 | "run":{ 22 | "query":{"field":{ "name":"query", "kind":"text", "order":1, "default":"", "description":"SQL with newlines and all." }}, 23 | "legacy":{"field":{ "name":"legacy", "kind":"boolean", "order":2, "default":true, "description":"Query type must match table and query format." }} 24 | } 25 | }} 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | 19 | # starthinker 20 | google-api-python-client 21 | google-auth 22 | google-auth-oauthlib 23 | google-auth-httplib2 24 | google-cloud-bigquery>=2.13 25 | google-cloud-vision 26 | pysftp 27 | TwitterAPI 28 | python-dateutil 29 | pytz 30 | psutil 31 | moviepy 32 | py-xlsx 33 | simple-salesforce 34 | smartsheet-python-sdk 35 | 36 | # starthinker_ui 37 | wheel 38 | gunicorn 39 | jsonpickle 40 | django 41 | psycopg2-binary 42 | contextlib2 43 | -------------------------------------------------------------------------------- /scripts/dcm_run.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2019-05-28", 4 | "license":"Licensed under the Apache License, Version 2.0", 5 | "copyright":"Copyright 2020 Google LLC", 6 | "icon":"donut_small", 7 | "from":["cm360"], 8 | "to":["cm360"], 9 | "title":"CM360 Report Run", 10 | "description":"Trigger a CM report run", 11 | "instructions":[ 12 | "Specify an account id.", 13 | "Specify either report name or report id to run." 14 | ], 15 | "authors":["sangnguyen@google.com"] 16 | }, 17 | "tasks":[ 18 | { "dcm":{ 19 | "auth":{"field":{ "name":"auth_read", "kind":"authentication", "order":1, "default":"user", "description":"Credentials used for reading data." }}, 20 | "report_run_only":true, 21 | "report":{ 22 | "account":{"field":{ "name":"account", "kind":"integer", "order":1, "default":"", "description":"CM network id." }}, 23 | "report_id":{"field":{ "name":"report_id", "kind":"integer", "order":2, "default":"", "description":"CM report id, empty if using name." }}, 24 | "name":{"field":{ "name":"report_name", "kind":"string", "order":3, "default":"", "description":"CM report name, empty if using id instead." }} 25 | } 26 | }} 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /starthinker_ui/project/urls.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | 19 | from django.conf.urls import url 20 | 21 | from starthinker_ui.project import views 22 | 23 | urlpatterns = [ 24 | url(r'^project/edit/(?P\d+)?/?$', 25 | views.project_edit, 26 | name='project.edit'), 27 | url(r'^project/delete/(?P\d+)?/?$', 28 | views.project_delete, 29 | name='project.delete'), 30 | url(r'^project/$', views.project_list, name='project.list'), 31 | ] 32 | -------------------------------------------------------------------------------- /starthinker_ui/templates/form_setup.html: -------------------------------------------------------------------------------- 1 | {% comment %} 2 | ########################################################################### 3 | # 4 | # Copyright 2020 Google LLC 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | ########################################################################### 19 | {% endcomment %} 20 | 21 | {% for section in form_setup.structure %} 22 |
23 |

{{ section.title }}

24 | {% if section.description %}

{{ section.description|safe }}

{% endif %} 25 | {% with fields=section.fields %} 26 | {% include "form_fields.html" %} 27 | {% endwith %} 28 |
29 |
30 | {% endfor %} 31 | -------------------------------------------------------------------------------- /tests/smartsheet.json: -------------------------------------------------------------------------------- 1 | { 2 | "setup":{ 3 | "license":"Licensed under the Apache License, Version 2.0", 4 | "copyright":"Copyright 2020 Google LLC" 5 | }, 6 | "tasks": [ 7 | { "dataset": { 8 | "auth": "service", 9 | "dataset": "SmartSheets" 10 | }}, 11 | { "include":{ 12 | "script":"scripts/smartsheet_report_to_bigquery.json", 13 | "parameters":{ 14 | "token":{"field":{ "name":"token", "kind":"string", "description":"Retrieve from SmartSheet account settings." }}, 15 | "report":{"field":{ "name":"report", "kind":"string", "description":"Retrieve from report properties." }}, 16 | "dataset":"Test", 17 | "table":"SmartSheet_Report" 18 | } 19 | }}, 20 | { "include":{ 21 | "script":"scripts/smartsheet_to_bigquery.json", 22 | "parameters":{ 23 | "token":{"field":{ "name":"token", "kind":"string", "description":"Retrieve from SmartSheet account settings." }}, 24 | "sheet":{"field":{ "name":"sheet", "kind":"string", "description":"Retrieve from sheet properties." }}, 25 | "dataset":"Test", 26 | "table":"SmartSheet_Sheet" 27 | } 28 | }}, 29 | { "test": { 30 | "assert":"Completed all tasks." 31 | }} 32 | ] 33 | } 34 | -------------------------------------------------------------------------------- /starthinker_ui/templates/input_checkbox_multiple.html: -------------------------------------------------------------------------------- 1 | {% comment %} 2 | ########################################################################### 3 | # 4 | # Copyright 2020 Google LLC 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | ########################################################################### 19 | {% endcomment %} 20 | 21 | {% load i18n %} 22 | 23 | {% for value, name in field.choices %} 24 |
25 | 31 |
32 | {% endfor %} 33 | -------------------------------------------------------------------------------- /starthinker_ui/ui/wsgi.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | """WSGI config for ui project. 19 | 20 | It exposes the WSGI callable as a module-level variable named ``application``. 21 | 22 | For more information on this file, see 23 | https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/ 24 | """ 25 | 26 | import os 27 | 28 | from django.core.wsgi import get_wsgi_application 29 | 30 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'starthinker_ui.ui.settings') 31 | 32 | application = get_wsgi_application() 33 | -------------------------------------------------------------------------------- /starthinker_ui/ui/urls.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | 19 | from django.conf import settings 20 | from django.conf.urls import url, include 21 | from django.conf.urls.static import static 22 | 23 | urlpatterns = [ 24 | url(r'', include('starthinker_ui.account.urls')), 25 | url(r'', include('starthinker_ui.recipe.urls')), 26 | url(r'', include('starthinker_ui.project.urls')), 27 | url(r'', include('starthinker_ui.website.urls')), 28 | ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) 29 | -------------------------------------------------------------------------------- /starthinker/airflow/operators/hello.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | 19 | import time 20 | 21 | from airflow import models 22 | from airflow.utils.decorators import apply_defaults 23 | 24 | 25 | class Hello(models.BaseOperator): 26 | 27 | @apply_defaults 28 | def __init__(self, say='', sleep=0, **kwargs): 29 | self.say = say 30 | self.sleep = sleep 31 | super(Hello, self).__init__(**kwargs) 32 | 33 | def execute(self, context): 34 | print(self.say) 35 | if self.sleep: 36 | time.sleep(self.sleep) 37 | -------------------------------------------------------------------------------- /tests/cm360_oculi.json: -------------------------------------------------------------------------------- 1 | { 2 | "setup":{ 3 | "license":"Licensed under the Apache License, Version 2.0", 4 | "copyright":"Copyright 2021 Google LLC" 5 | }, 6 | "tasks": [ 7 | { "dataset":{ 8 | "auth":"service", 9 | "dataset":{"field":{ "name":"test_run_id", "kind":"string", "description":"The dataset to use for the test.", "prefix":"StarThinker_Test_Oculi_CM_", "default": "Manual"}}, 10 | "clear":true 11 | }}, 12 | { "include":{ 13 | "script":"scripts/cm360_oculi.json", 14 | "parameters":{ 15 | "auth_cm":"user", 16 | "auth_bigquery":"service", 17 | "limit":500, 18 | "account":{"field":{ "name":"account", "kind":"integer", "description":"CM360 Account Identifier" }}, 19 | "recipe_slug":{"field":{ "name":"test_run_id", "kind":"string", "description":"The dataset to use for the test.", "prefix":"StarThinker_Test_Oculi_CM_", "default": "Manual"}} 20 | } 21 | }}, 22 | { "dataset":{ 23 | "auth":"service", 24 | "dataset":{"field":{ "name":"test_run_id", "kind":"string", "description":"The dataset to use for the test.", "prefix":"StarThinker_Test_Oculi_CM_", "default": "Manual"}}, 25 | "delete":true 26 | }}, 27 | { "test": { 28 | "assert":"Completed all tasks." 29 | }} 30 | ] 31 | } 32 | -------------------------------------------------------------------------------- /scripts/sheets_copy.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2018-10-02", 4 | "license":"Licensed under the Apache License, Version 2.0", 5 | "copyright":"Copyright 2020 Google LLC", 6 | "icon":"view_column", 7 | "title":"Sheet Copy", 8 | "description":"Copy tab from a sheet to a sheet.", 9 | "test":"https://github.com/google/starthinker/blob/master/tests/sheets.json", 10 | "from":["sheets"], 11 | "to":["sheets"], 12 | "instructions":[ 13 | "Provide the full edit URL for both sheets.", 14 | "Provide the tab name for both sheets.", 15 | "The tab will only be copied if it does not already exist." 16 | ], 17 | "authors":["kenjora@google.com"] 18 | }, 19 | "tasks":[ 20 | { "sheets":{ 21 | "auth":{"field":{ "name":"auth_read", "kind":"authentication", "order":1, "default":"user", "description":"Credentials used for reading data." }}, 22 | "template":{ 23 | "sheet":{"field":{ "name":"from_sheet", "kind":"string", "order":1, "default":"" }}, 24 | "tab":{"field":{ "name":"from_tab", "kind":"string", "order":2, "default":"" }} 25 | }, 26 | "sheet":{"field":{ "name":"to_sheet", "kind":"string", "order":3, "default":"" }}, 27 | "tab":{"field":{ "name":"to_tab", "kind":"string", "order":4, "default":"" }} 28 | }} 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /starthinker/task/bucket/run.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | 19 | from starthinker.util.storage import bucket_create, bucket_access 20 | 21 | def bucket(config, task): 22 | if config.verbose: 23 | print('BUCKET', task['bucket']) 24 | 25 | bucket_create( 26 | config, 27 | task['auth'], 28 | config.project, 29 | task['bucket'] 30 | ) 31 | 32 | bucket_access( 33 | config, 34 | task['auth'], 35 | config.project, 36 | task['bucket'], 37 | emails=task.get('emails', []), 38 | groups=task.get('groups', []) 39 | ) 40 | -------------------------------------------------------------------------------- /starthinker/task/weather_gov/check.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | 19 | from starthinker.util.data import get_rows 20 | 21 | 22 | def weather_gov_test(config, task): 23 | print('testing weather_gov connector') 24 | 25 | if 'verify' in task['weather_gov']: 26 | rows = get_rows(config, task['auth'], 27 | task['weather_gov']['verify']['read']) 28 | station_ids = task['weather_gov']['verify']['station_ids'] 29 | 30 | for idx, row in enumerate(rows): 31 | if row[1] != station_ids[idx]: 32 | raise 'Station weather data not loaded correctly' 33 | -------------------------------------------------------------------------------- /scripts/dbm.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2017-08-14", 4 | "license":"Licensed under the Apache License, Version 2.0", 5 | "copyright":"Copyright 2020 Google LLC", 6 | "icon":"donut_large", 7 | "title":"DV360 Report", 8 | "description":"Create a DV360 report.", 9 | "from":["dv360"], 10 | "to":["dv360"], 11 | "instructions":[ 12 | "Reference field values from the [DV360 API](https://developers.google.com/bid-manager/v1/reports) to build a report.", 13 | "Copy and paste the JSON definition of a report, [sample for reference](https://github.com/google/starthinker/blob/master/tests/dbm_to_bigquery.json).", 14 | "The report is only created, a seperate script is required to move the data.", 15 | "To reset a report, delete it from DV360 reporting." 16 | ], 17 | "authors":["kenjora@google.com"] 18 | }, 19 | "tasks":[ 20 | { "dbm":{ 21 | "auth":{"field":{ "name":"auth_read", "kind":"authentication", "order":1, "default":"user", "description":"Credentials used for reading data." }}, 22 | "report": {"field":{ "name":"report", "kind":"json", "order":1, "default":"{}", "description":"Report body and filters."}}, 23 | "delete":{"field":{ "name":"delete", "kind":"boolean", "order":2, "default":false, "description":"If report exists, delete it before creating a new one." }} 24 | }} 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /scripts/archive.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2017-08-14", 4 | "license":"Licensed under the Apache License, Version 2.0", 5 | "copyright":"Copyright 2020 Google LLC", 6 | "icon":"archive", 7 | "title":"Storage Archive", 8 | "description":"Wipe old information from a Storage bucket based on last update time.", 9 | "test":"https://github.com/google/starthinker/blob/master/tests/archive.json", 10 | "from":["storage"], 11 | "to":["storage"], 12 | "instructions":[ 13 | "Specify how many days back to retain data and which buckets and paths to purge.", 14 | "Everything under a path will be moved to archive or deleted depending on your choice." 15 | ], 16 | "authors":["kenjora@google.com"] 17 | }, 18 | "tasks":[ 19 | { "archive":{ 20 | "auth":{"field":{ "name":"auth_write", "kind":"authentication", "order":1, "default":"service", "description":"Credentials used for writing data." }}, 21 | "days":{"field":{ "name":"archive_days", "kind":"integer", "order":1, "default":7 }}, 22 | "storage": { 23 | "bucket":{"field":{ "name":"archive_bucket", "kind":"string", "order":2, "default":"" }}, 24 | "path":{"field":{ "name":"archive_path", "kind":"string", "order":3, "default":"" }} 25 | }, 26 | "delete":{"field":{ "name":"archive_delete", "kind":"boolean", "order":4, "default":false }} 27 | }} 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /scripts/bucket.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2017-08-15", 4 | "released":"2017-08-15", 5 | "license":"Licensed under the Apache License, Version 2.0", 6 | "copyright":"Copyright 2020 Google LLC", 7 | "icon":"storage", 8 | "from":["storage"], 9 | "to":["storage"], 10 | "title":"Storage Bucket", 11 | "description":"Create and permission a bucket in Storage.", 12 | "instructions":[ 13 | "Specify the name of the bucket and who will have owner permissions.", 14 | "Existing buckets are preserved.", 15 | "Adding a permission to the list will update the permissions but removing them will not.", 16 | "You have to manualy remove grants." 17 | ], 18 | "authors":["kenjora@google.com"] 19 | }, 20 | "tasks":[ 21 | { "bucket":{ 22 | "auth":{"field":{ "name":"auth_write", "kind":"authentication", "order":1, "default":"service", "description":"Credentials used for writing data." }}, 23 | "bucket":{"field":{ "name":"bucket_bucket", "kind":"string", "order":2, "default":"", "description":"Name of Google Cloud Bucket to create." }}, 24 | "emails":{"field":{ "name":"bucket_emails", "kind":"string_list", "order":3, "default":"", "description":"Comma separated emails." }}, 25 | "groups":{"field":{ "name":"bucket_groups", "kind":"string_list", "order":4, "default":"", "description":"Comma separated groups." }} 26 | }} 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /scripts/bigquery_function.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2020-07-07", 4 | "license":"Licensed under the Apache License, Version 2.0", 5 | "copyright":"Copyright 2020 Google LLC", 6 | "icon":"addchart", 7 | "title":"BigQuery Function", 8 | "description":"Add a custom function or table to a dataset.", 9 | "test":"https://github.com/google/starthinker/blob/master/tests/bigquery_function.json", 10 | "to":["bigquery"], 11 | "from":["bigquery"], 12 | "instructions":[ 13 | "Specify the dataset, and the function or table will be added.", 14 | "Pearson Significance Test: Check if a correlation is significant.", 15 | "RGB To HSV: Convert color values for analysis." 16 | ], 17 | "authors":["kenjora@google.com"] 18 | }, 19 | "tasks":[ 20 | { "bigquery":{ 21 | "auth":{"field":{ "name":"auth", "kind":"authentication", "order":0, "default":"service", "description":"Credentials used for writing function." }}, 22 | "function":{"field":{ 23 | "name":"function", 24 | "kind":"choice", 25 | "order":1, 26 | "choices":["Pearson Significance Test", "RGB To HSV"], 27 | "default":"Pearson Significance Test", 28 | "description":"Function or table to create." 29 | }}, 30 | "to":{ 31 | "dataset":{"field":{ "name":"dataset", "kind":"string", "order":1, "default":"", "description":"Existing BigQuery dataset." }} 32 | } 33 | }} 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /scripts/dataset.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2017-08-14", 4 | "license":"Licensed under the Apache License, Version 2.0", 5 | "copyright":"Copyright 2020 Google LLC", 6 | "icon":"folder", 7 | "from":["bigquery"], 8 | "to":["bigquery"], 9 | "title":"BigQuery Dataset", 10 | "description":"Create and permission a dataset in BigQuery.", 11 | "test":"https://github.com/google/starthinker/blob/master/tests/dataset.json", 12 | "instructions":[ 13 | "Specify the name of the dataset.", 14 | "If dataset exists, it is inchanged.", 15 | "Add emails and / or groups to add read permission.", 16 | "CAUTION: Removing permissions in StarThinker has no effect.", 17 | "CAUTION: To remove permissions you have to edit the dataset." 18 | ], 19 | "authors":["kenjora@google.com"] 20 | }, 21 | "tasks":[ 22 | { "dataset":{ 23 | "auth":{"field":{ "name":"auth_write", "kind":"authentication", "order":1, "default":"service", "description":"Credentials used for writing data." }}, 24 | "dataset":{"field":{ "name":"dataset_dataset", "kind":"string", "order":1, "default":"", "description":"Name of Google BigQuery dataset to create." }}, 25 | "emails":{"field":{ "name":"dataset_emails", "kind":"string_list", "order":2, "default":[], "description":"Comma separated emails." }}, 26 | "groups":{"field":{ "name":"dataset_groups", "kind":"string_list", "order":3, "default":[], "description":"Comma separated groups." }} 27 | }} 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /scripts/bigquery_view.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2017-08-14", 4 | "license":"Licensed under the Apache License, Version 2.0", 5 | "copyright":"Copyright 2020 Google LLC", 6 | "icon":"arrow_forward", 7 | "title":"BigQuery Query To View", 8 | "description":"Create a BigQuery view.", 9 | "from":["bigquery"], 10 | "to":["bigquery"], 11 | "instructions":[ 12 | "Specify a single query and choose legacy or standard mode.", 13 | "For PLX use: SELECT * FROM [plx.google:FULL_TABLE_NAME.all] WHERE...", 14 | "If the view exists, it is unchanged, delete it manually to re-create." 15 | ], 16 | "authors":["kenjora@google.com"] 17 | }, 18 | "tasks":[ 19 | { "bigquery":{ 20 | "auth":{"field":{ "name":"auth_read", "kind":"authentication", "order":1, "default":"user", "description":"Credentials used for reading data." }}, 21 | "from":{ 22 | "query":{"field":{ "name":"query", "kind":"text", "order":1, "default":"", "description":"SQL with newlines and all." }}, 23 | "legacy":{"field":{ "name":"legacy", "kind":"boolean", "order":4, "default":true, "description":"Query type must match source tables." }} 24 | }, 25 | "to":{ 26 | "dataset":{"field":{ "name":"dataset", "kind":"string", "order":2, "default":"", "description":"Existing BigQuery dataset." }}, 27 | "view":{"field":{ "name":"view", "kind":"string", "order":3, "default":"", "description":"View to create from this query." }} 28 | } 29 | }} 30 | ] 31 | } 32 | -------------------------------------------------------------------------------- /scripts/bigquery_query.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2017-08-14", 4 | "license":"Licensed under the Apache License, Version 2.0", 5 | "copyright":"Copyright 2020 Google LLC", 6 | "icon":"arrow_forward", 7 | "title":"BigQuery Query To Table", 8 | "description":"Save query results into a BigQuery table.", 9 | "to":["bigquery"], 10 | "from":["bigquery"], 11 | "instructions":[ 12 | "Specify a single query and choose legacy or standard mode.", 13 | "For PLX use user authentication and: SELECT * FROM [plx.google:FULL_TABLE_NAME.all] WHERE...", 14 | "Every time the query runs it will overwrite the table." 15 | ], 16 | "authors":["kenjora@google.com"] 17 | }, 18 | "tasks":[ 19 | { "bigquery":{ 20 | "auth":{"field":{ "name":"auth_write", "kind":"authentication", "order":1, "default":"service", "description":"Credentials used for writing data." }}, 21 | "from":{ 22 | "query":{"field":{ "name":"query", "kind":"text", "order":1, "default":"", "description":"SQL with newlines and all." }}, 23 | "legacy":{"field":{ "name":"legacy", "kind":"boolean", "order":4, "default":true, "description":"Query type must match source tables." }} 24 | }, 25 | "to":{ 26 | "dataset":{"field":{ "name":"dataset", "kind":"string", "order":2, "default":"", "description":"Existing BigQuery dataset." }}, 27 | "table":{"field":{ "name":"table", "kind":"string", "order":3, "default":"", "description":"Table to create from this query." }} 28 | } 29 | }} 30 | ] 31 | } 32 | -------------------------------------------------------------------------------- /starthinker/task/drive/run.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | """Handler that executes { "drive":{...}} task in recipe JSON. 19 | 20 | This script perfroms various drive operations like copy and create. 21 | """ 22 | 23 | from starthinker.util.drive import file_copy, file_create, file_delete 24 | 25 | 26 | def drive(config, task): 27 | if config.verbose: 28 | print('Drive') 29 | 30 | if 'delete' in task: 31 | if config.verbose: 32 | print('Drive Delete', task['delete']) 33 | file_delete(config, task['auth'], task['delete']) 34 | 35 | if 'copy' in task: 36 | if config.verbose: 37 | print('Drive Copy', task['copy']['source'], 38 | task['copy']['destination']) 39 | file_copy(config, task['auth'], task['copy']['source'], 40 | task['copy']['destination']) 41 | -------------------------------------------------------------------------------- /starthinker/util/dynamite.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | 19 | import json 20 | from urllib.request import Request, urlopen 21 | 22 | 23 | def dynamite_write(room, key, token, text): 24 | url = 'https://chat.googleapis.com/v1/spaces/%s/messages?key=%s&token=%s' % ( 25 | room, key, token) 26 | data = json.dumps({ 27 | 'sender': { 28 | 'displayName': 29 | 'StarThinker', 30 | 'avatarUrl': 31 | 'https://www.gstatic.com/images/icons/material/system/2x/chat_googblue_48dp.png' 32 | }, 33 | 'text': text 34 | }) 35 | f = urlopen( 36 | urllib.request.Request(url, data, { 37 | 'Content-Type': 'application/json', 38 | 'Content-Length': len(data) 39 | })) 40 | response = f.read() 41 | f.close() 42 | return response 43 | -------------------------------------------------------------------------------- /scripts/test.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2019-05-03", 4 | "license":"Licensed under the Apache License, Version 2.0", 5 | "copyright":"Copyright 2020 Google LLC", 6 | "ui":false, 7 | "icon":"notifications", 8 | "title":"Test Script", 9 | "description":"Used by tests.", 10 | "instructions":[ 11 | "This should be called by the tests scripts only.", 12 | "When run will generate a say hello log." 13 | ], 14 | "authors":["kenjora@google.com"] 15 | }, 16 | "setup":{ 17 | "day":["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"], 18 | "hour":[1, 3, 23] 19 | }, 20 | "tasks":[ 21 | { "hello":{ 22 | "auth":"user", 23 | "hour":[1], 24 | "say":"Hello At 1", 25 | "sleep":0 26 | }}, 27 | { "hello":{ 28 | "auth":"user", 29 | "hour":[3], 30 | "say":"Hello At 3", 31 | "sleep":0 32 | }}, 33 | { "hello":{ 34 | "auth":"user", 35 | "hour":[], 36 | "say":"Hello Manual", 37 | "sleep":0 38 | }}, 39 | { "hello":{ 40 | "auth":"user", 41 | "hour":[23], 42 | "say":"Hello At 23 Sleep", 43 | "sleep":30 44 | }}, 45 | { "hello":{ 46 | "auth":"user", 47 | "say":"Hello At Anytime", 48 | "sleep":0 49 | }}, 50 | { "hello":{ 51 | "auth":"user", 52 | "hour":[1, 3, 23], 53 | "say":"Hello At 1, 3, 23", 54 | "sleep":0 55 | }}, 56 | { "hello":{ 57 | "auth":"user", 58 | "hour":[3], 59 | "say":"Hello At 3 Reordered", 60 | "sleep":0 61 | }} 62 | ] 63 | } 64 | -------------------------------------------------------------------------------- /tutorials/data_schemas.md: -------------------------------------------------------------------------------- 1 | # List Of StarThinker Schema Definitions 2 | 3 | Various StarThinker recipies use schemas to ensure data transfered into BigQuery or another database is stable. 4 | This document gives a collective summary of most 5 | 6 | ### Why you should be using explicit schemas in all data moves... 7 | 8 | Defining a schema explicitly removes ambiguity and flags errors early preventing down stream time sinks. Data can change, 9 | a column of NULLs can be and INTEGER or a STRING, but your down stream formula can only SUM one of them. 10 | 11 | ## List Of Schemas As Python Dictionaries 12 | - [Campaign Manager - Report And DT Schema](https://github.com/google/starthinker/tree/master/starthinker/task/dt/schema) - Used by [dt](https://github.com/google/starthinker/tree/master/starthinker/task/dt) and [dcm](https://github.com/google/starthinker/tree/master/starthinker/task/dcm) task. 13 | - [Campaign Manager - List Schema](https://github.com/google/starthinker/tree/master/starthinker/task/dcm_api/schema) - Used by [dcm_api](https://github.com/google/starthinker/tree/master/starthinker/task/dcm_api) task. 14 | - [DV360 - Entity Read Schema](https://github.com/google/starthinker/tree/master/starthinker/task/entity/schema)- Used by [entity](https://github.com/google/starthinker/tree/master/starthinker/task/entity) task. 15 | - [DV360 - SDF Files](https://github.com/google/starthinker/tree/master/starthinker/util/dcm/schema) - Used by [sdf](https://github.com/google/starthinker/tree/master/starthinker/task/sdf) task. 16 | 17 | --- 18 | © 2020 Google Inc. - Apache License, Version 2.0 19 | -------------------------------------------------------------------------------- /starthinker/task/sdf/run.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | 19 | from starthinker.util.sdf import sdf_download, sdf_to_bigquery 20 | 21 | 22 | def sdf(config, task): 23 | if config.verbose: 24 | print('SDF') 25 | 26 | # Download sdf files 27 | sdf_zip_file = sdf_download(config, task['auth'], task['version'], 28 | task['partner_id'], 29 | task['file_types'], 30 | task['filter_type'], 31 | task['read']['filter_ids']) 32 | 33 | # Load data into BigQuery 34 | sdf_to_bigquery(config, task['auth'], sdf_zip_file, config.project, task['dataset'], 35 | task['time_partitioned_table'], 36 | task['create_single_day_table'], 37 | task.get('table_suffix', '')) 38 | -------------------------------------------------------------------------------- /scripts/hello.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2019-05-03", 4 | "license":"Licensed under the Apache License, Version 2.0", 5 | "copyright":"Copyright 2020 Google LLC", 6 | "ui":false, 7 | "icon":"notifications", 8 | "title":"Say Hello", 9 | "description":"Recipe template for say hello.", 10 | "instructions":[ 11 | "This should be called for testing only." 12 | ], 13 | "authors":["kenjora@google.com"] 14 | }, 15 | "setup":{ 16 | "day":["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"] 17 | }, 18 | "tasks":[ 19 | { "hello":{ 20 | "auth":{"field":{ "name":"auth_read", "kind":"authentication", "order":1, "default":"user", "description":"Credentials used for reading data." }}, 21 | "say":{"field":{ "name":"say_first", "kind":"string", "order":1, "default":"Hello Once", "description":"Type in a greeting." }}, 22 | "error":{"field":{ "name":"error", "kind":"string", "order":3, "default":"", "description":"Optional error for testing." }}, 23 | "sleep":{"field":{ "name":"sleep", "kind":"integer", "order":4, "default":0, "description":"Seconds to sleep." }} 24 | }}, 25 | { "hello":{ 26 | "auth":{"field":{ "name":"auth_read", "kind":"authentication", "order":1, "default":"user", "description":"Credentials used for reading data." }}, 27 | "say":{"field":{ "name":"say_second", "kind":"string", "order":1, "default":"Hello Twice", "description":"Type in a greeting." }}, 28 | "sleep":{"field":{ "name":"sleep", "kind":"integer", "order":4, "default":0, "description":"Seconds to sleep." }} 29 | }} 30 | ] 31 | } 32 | -------------------------------------------------------------------------------- /starthinker/task/salesforce/run.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | """Handler that executes { "salesforce":{...}} task in recipe JSON. 19 | 20 | This script translates JSON instructions into operations on salesforce 21 | reporting. 22 | 23 | """ 24 | 25 | from starthinker.util.data import put_rows 26 | from starthinker.util.salesforce import authenticate, query 27 | 28 | 29 | def salesforce(config, task): 30 | if config.verbose: 31 | print('Salesforce') 32 | 33 | sf = authenticate( 34 | task['domain'], 35 | task['client'], 36 | task['secret'], 37 | task['username'], 38 | task['password'] 39 | ) 40 | 41 | if 'query' in task: 42 | rows = query(sf, task['query']) 43 | if rows: 44 | put_rows( 45 | config, 46 | task['auth'], 47 | task['out'], 48 | rows 49 | ) 50 | -------------------------------------------------------------------------------- /starthinker_ui/recipe/management/commands/recipe_usage.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2022 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | 19 | from django.core.management.base import BaseCommand, CommandError 20 | from django.conf import settings 21 | 22 | from starthinker_ui.account.models import Account 23 | 24 | class Command(BaseCommand): 25 | help = 'Prints recipe count and age by account.' 26 | 27 | def handle(self, *args, **kwargs): 28 | usage = [] 29 | 30 | for account in Account.objects.all(): 31 | usage.append({ 32 | 'email':account.email, 33 | 'recipes':list(account.recipe_set.all().values_list('birthday', flat=True)) 34 | }) 35 | 36 | usage.sort(key=lambda u: len(u['recipes']), reverse=True) 37 | 38 | for u in usage: 39 | print ('{}, {}, {}'.format(u['email'], len(u['recipes']), max(u['recipes']) if u['recipes'] else '')) 40 | -------------------------------------------------------------------------------- /scripts/dcm_to_sheets.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2018-01-29", 4 | "license":"Licensed under the Apache License, Version 2.0", 5 | "copyright":"Copyright 2020 Google LLC", 6 | "icon":"donut_small", 7 | "from":["cm360"], 8 | "to":["sheets"], 9 | "title":"CM360 Report To Sheets", 10 | "description":"Move existing CM report into a Sheet tab.", 11 | "test":"https://github.com/google/starthinker/blob/master/tests/dcm_to_sheets.json", 12 | "instructions":[ 13 | "Specify an account id.", 14 | "Specify either report name or report id to move a report.", 15 | "The most recent valid file will be moved to the sheet.", 16 | "Schema is pulled from the official CM specification." 17 | ], 18 | "authors":["kenjora@google.com"] 19 | }, 20 | "tasks":[ 21 | { "dcm":{ 22 | "auth":{"field":{ "name":"auth_read", "kind":"authentication", "order":1, "default":"user", "description":"Credentials used for reading data." }}, 23 | "report":{ 24 | "account":{"field":{ "name":"account", "kind":"integer", "order":2, "default":"" }}, 25 | "report_id":{"field":{ "name":"report_id", "kind":"integer", "order":3, "default":"" }}, 26 | "name":{"field":{ "name":"report_name", "kind":"string", "order":4, "default":"" }} 27 | }, 28 | "out":{ 29 | "sheets":{ 30 | "sheet":{"field":{ "name":"sheet", "kind":"string", "order":5, "default":"" }}, 31 | "tab":{"field":{ "name":"tab", "kind":"string", "order":6, "default":"" }}, 32 | "range":"A1" 33 | } 34 | } 35 | }} 36 | ] 37 | } 38 | -------------------------------------------------------------------------------- /starthinker/task/sa/run.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2021 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | 19 | """Handler that executes { "sa":{...}} task in recipe JSON. 20 | 21 | Connects the Google Search Ads Reporting API to recipe JSON. This task 22 | is required because the reporting response needs additional processing. 23 | """ 24 | 25 | from starthinker.util.data import put_rows 26 | from starthinker.util.sa import SA_Report 27 | 28 | 29 | def sa(config, task): 30 | if config.verbose: 31 | print('SA') 32 | 33 | report = SA_Report(config, task['auth']) 34 | report.request(task['body']) 35 | rows = report.get_rows() 36 | 37 | if 'bigquery' in task.get('out', {}): 38 | task['out']['bigquery']['schema'] = report.get_schema() 39 | task['out']['bigquery']['format'] = 'CSV' 40 | 41 | return put_rows( 42 | config, 43 | task['auth'], 44 | task['out'], 45 | rows 46 | ) 47 | -------------------------------------------------------------------------------- /starthinker_ui/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ########################################################################### 4 | # 5 | # Copyright 2020 Google LLC 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | ########################################################################### 20 | 21 | import os 22 | import sys 23 | 24 | if __name__ == '__main__': 25 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ui.settings') 26 | try: 27 | from django.core.management import execute_from_command_line 28 | except ImportError: 29 | # The above import may fail for some other reason. Ensure that the 30 | # issue is really that Django is missing to avoid masking other 31 | # exceptions on Python 2. 32 | try: 33 | import django 34 | except ImportError: 35 | raise ImportError( 36 | "Couldn't import Django. Are you sure it's installed and " 37 | 'available on your PYTHONPATH environment variable? Did you ' 38 | 'forget to activate a virtual environment?') 39 | raise 40 | execute_from_command_line(sys.argv) 41 | -------------------------------------------------------------------------------- /scripts/dbm_to_sheets.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2018-01-29", 4 | "license":"Licensed under the Apache License, Version 2.0", 5 | "copyright":"Copyright 2020 Google LLC", 6 | "icon":"donut_large", 7 | "from":["dv360"], 8 | "to":["sheets"], 9 | "title":"DV360 Report To Sheets", 10 | "description":"Move existing DV360 report into a Sheets tab.", 11 | "test":"https://github.com/google/starthinker/blob/master/tests/dbm_to_sheets.json", 12 | "instructions":[ 13 | "Specify either report name or report id to move a report.", 14 | "The most recent valid file will be moved to the sheet." 15 | ], 16 | "authors":["kenjora@google.com"] 17 | }, 18 | "tasks":[ 19 | { "dbm":{ 20 | "auth":{"field":{ "name":"auth_read", "kind":"authentication", "order":1, "default":"user", "description":"Credentials used for reading data." }}, 21 | "report":{ 22 | "report_id":{"field":{ "name":"report_id", "kind":"integer", "order":1, "default":"", "description":"DV360 report ID given in UI, not needed if name used." }}, 23 | "name":{"field":{ "name":"report_name", "kind":"string", "order":2, "default":"", "description":"Name of report, not needed if ID used." }} 24 | }, 25 | "out":{ 26 | "sheets":{ 27 | "sheet":{"field":{ "name":"sheet", "kind":"string", "order":3, "default":"", "description":"Full URL to sheet being written to." }}, 28 | "tab":{"field":{ "name":"tab", "kind":"string", "order":4, "default":"", "description":"Existing tab in sheet to write to." }}, 29 | "range":"A1" 30 | } 31 | } 32 | }} 33 | ] 34 | } 35 | -------------------------------------------------------------------------------- /scripts/dcm_to_storage.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2017-08-14", 4 | "license":"Licensed under the Apache License, Version 2.0", 5 | "copyright":"Copyright 2020 Google LLC", 6 | "icon":"donut_small", 7 | "title":"CM360 Report To Storage", 8 | "description":"Move existing CM report into a Storage bucket.", 9 | "from":["cm360"], 10 | "to":["storage"], 11 | "instructions":[ 12 | "Specify an account id.", 13 | "Specify either report name or report id to move a report.", 14 | "The most recent file will be moved to the bucket.", 15 | "Schema is pulled from the official CM specification." 16 | ], 17 | "authors":["kenjora@google.com"] 18 | }, 19 | "tasks":[ 20 | { "dcm":{ 21 | "auth":{"field":{ "name":"auth_read", "kind":"authentication", "order":1, "default":"user", "description":"Credentials used for reading data." }}, 22 | "report":{ 23 | "account":{"field":{ "name":"account", "kind":"integer", "order":2, "default":"" }}, 24 | "report_id":{"field":{ "name":"report_id", "kind":"integer", "order":3, "default":"" }}, 25 | "name":{"field":{ "name":"report_name", "kind":"string", "order":4, "default":"" }} 26 | }, 27 | "out":{ 28 | "storage":{ 29 | "auth":{"field":{ "name":"auth_write", "kind":"authentication", "order":1, "default":"service", "description":"Credentials used for writing data." }}, 30 | "bucket":{"field":{ "name":"bucket", "kind":"string", "order":5, "default":"" }}, 31 | "path":{"field":{ "name":"path", "kind":"string", "order":6, "default":"CM_Report" }} 32 | } 33 | } 34 | }} 35 | ] 36 | } 37 | -------------------------------------------------------------------------------- /starthinker/task/traffic/check.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | 19 | from starthinker.util.google_api import API_DCM 20 | from starthinker.util.cm import get_profile_for_api 21 | 22 | 23 | def bulkdozer_test(config, task): 24 | print('testing bulkdozer') 25 | 26 | if 'verify' in task['traffic']: 27 | is_admin, profile_id = get_profile_for_api( 28 | config, task['auth'], task['traffic']['account_id']) 29 | 30 | for entity in task['traffic']['verify']: 31 | service = getattr( 32 | API_DCM(config, task['auth'], internal=is_admin), entity['type']) 33 | cm_entity = service().get(profileId=profile_id, id=entity['id']).execute() 34 | 35 | values = entity['values'] 36 | 37 | for key in values: 38 | if values[key] != cm_entity[key]: 39 | raise ValueError('%s %s expected to be %s, was %s' % (entity['type'], key, values[key], cm_entity[key])) 40 | -------------------------------------------------------------------------------- /starthinker_ui/recipe/management/commands/autoscale.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | 19 | import json 20 | 21 | from django.core.management.base import BaseCommand, CommandError 22 | 23 | from starthinker_ui.recipe.views import autoscale 24 | from starthinker_ui.recipe.models import Recipe, utc_milliseconds, JOB_LOOKBACK_MS 25 | 26 | 27 | class Command(BaseCommand): 28 | help = 'Autoscale workers.' 29 | 30 | def handle(self, *args, **kwargs): 31 | print(json.dumps(json.loads(autoscale(None).content), indent=2)) 32 | 33 | for recipe in Recipe.objects.filter( 34 | active=True, job_utm__lt=utc_milliseconds()).exclude(job_utm=0): 35 | print(recipe.id, recipe.name, recipe.get_days()) 36 | print('---') 37 | 38 | for recipe in Recipe.objects.filter( 39 | active=True, worker_utm__gte=utc_milliseconds() - JOB_LOOKBACK_MS): 40 | print(recipe.id, recipe.name, recipe.worker_uid) 41 | -------------------------------------------------------------------------------- /starthinker/task/ga/run.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | 19 | """Handler that executes { "ga":{...}} task in recipe JSON. 20 | 21 | Connects the Google Analytics Reporting API to recipe JSON. This task 22 | is required because the reporting response needs additional processing. 23 | """ 24 | 25 | from starthinker.util.data import put_rows 26 | from starthinker.util.ga import GA_Report 27 | 28 | 29 | def ga(config, task): 30 | if config.verbose: 31 | print('GA') 32 | 33 | report = GA_Report( 34 | config, 35 | task['auth'], 36 | **task['kwargs'] 37 | ) 38 | 39 | # be sure to call before get_schema 40 | rows = report.get_rows() 41 | 42 | if 'bigquery' in task.get('out', {}): 43 | task['out']['bigquery']['schema'] = report.get_schema() 44 | task['out']['bigquery']['format'] = 'JSON' 45 | 46 | return put_rows( 47 | config, 48 | task['auth'], 49 | task['out'], 50 | rows 51 | ) 52 | -------------------------------------------------------------------------------- /scripts/dbm_to_storage.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2017-08-14", 4 | "license":"Licensed under the Apache License, Version 2.0", 5 | "copyright":"Copyright 2020 Google LLC", 6 | "icon":"donut_large", 7 | "from":["dv360"], 8 | "to":["storage"], 9 | "title":"DV360 Report To Storage", 10 | "description":"Move existing DV360 report into a Storage bucket.", 11 | "instructions":[ 12 | "Specify either report name or report id to move a report.", 13 | "The most recent valid file will be moved to the bucket." 14 | ], 15 | "authors":["kenjora@google.com"] 16 | }, 17 | "tasks":[ 18 | { "dbm":{ 19 | "auth":{"field":{ "name":"auth_read", "kind":"authentication", "order":1, "default":"user", "description":"Credentials used for reading data." }}, 20 | "report":{ 21 | "report_id":{"field":{ "name":"dbm_report_id", "kind":"integer", "order":1, "default":"", "description":"DV360 report ID given in UI, not needed if name used." }}, 22 | "name":{"field":{ "name":"dbm_report_name", "kind":"string", "order":2, "default":"", "description":"Name of report, not needed if ID used." }} 23 | }, 24 | "out":{ 25 | "storage":{ 26 | "auth":{"field":{ "name":"auth_write", "kind":"authentication", "order":1, "default":"service", "description":"Credentials used for writing data." }}, 27 | "bucket":{"field":{ "name":"dbm_bucket", "kind":"string", "order":3, "default":"", "description":"Google cloud bucket." }}, 28 | "path":{"field":{ "name":"dbm_path", "kind":"string", "order":4, "default":"", "description":"Path and filename to write to." }} 29 | } 30 | } 31 | }} 32 | ] 33 | } 34 | -------------------------------------------------------------------------------- /starthinker_ui/recipe/models_fields.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | ########################################################################### 4 | # 5 | # Copyright 2020 Google LLC 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | ########################################################################### 20 | 21 | import json 22 | 23 | from django.db import models 24 | 25 | 26 | class JsonField(models.TextField): 27 | 28 | # python to SQL 29 | def get_prep_value(self, value): 30 | if value is None: 31 | return None 32 | elif isinstance(value, str): 33 | return value 34 | else: 35 | try: 36 | return json.dumps(value) 37 | except: raise ValidationError(_('Bad JSON string.')) 38 | 39 | # SQL to python 40 | def from_db_value(self, value, expression, connection, context): 41 | return json.loads(value) 42 | 43 | # any source to python 44 | def to_python(self, value): 45 | if value is None: 46 | return None 47 | if isinstance(value, str): 48 | try: 49 | return json.loads(value) 50 | except: raise ValidationError(_('Bad JSON string.')) 51 | else: 52 | return value 53 | -------------------------------------------------------------------------------- /scripts/plx_to_bigquery.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2017-08-14", 4 | "icon":"account_balance", 5 | "title":"PLX To BigQuery", 6 | "description":"Save PLX query results into a BigQuery table.", 7 | "from":["plx"], 8 | "to":["bigquery"], 9 | "instructions":[ 10 | "Each PLX BigQuery Bridge [now requires per table approval](https://docs.google.com/document/d/1ok5lzaeb0GMbNomsIjQqCevSzcSUpDUGrcFJZmHP9zU/).", 11 | "SELECT * FROM [plx.google:FULL_TABLE_NAME.all] WHERE...", 12 | "Specify a single query and choose legacy or standard mode.", 13 | "Every time the query runs it will overwrite the table." 14 | ], 15 | "authors":["kenjora@google.com"] 16 | }, 17 | "tasks":[ 18 | { "bigquery":{ 19 | "auth":{"field":{ "name":"auth_read", "kind":"authentication", "order":1, "default":"user", "description":"Credentials used for reading data." }}, 20 | "from":{ 21 | "query":{"field":{ "name":"query", "kind":"text", "order":1, "default":"SELECT * FROM [plx.google:FULL_TABLE_NAME.all] WHERE...", "description":"Query to run on PLX as this user." }}, 22 | "legacy":{"field":{ "name":"legacy", "kind":"boolean", "order":4, "default":true, "description":"Legacy Or Standard SQL." }} 23 | }, 24 | "to":{ 25 | "auth":{"field":{ "name":"auth_write", "kind":"authentication", "order":1, "default":"service", "description":"Credentials used for writing data." }}, 26 | "dataset":{"field":{ "name":"dataset", "kind":"string", "order":2, "default":"", "description":"Existing BigQuery dataset." }}, 27 | "table":{"field":{ "name":"table", "kind":"string", "order":3, "default":"", "description":"Table to create from this query." }} 28 | } 29 | }} 30 | ] 31 | } 32 | -------------------------------------------------------------------------------- /scripts/bigquery_storage.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2018-03-15", 4 | "license":"Licensed under the Apache License, Version 2.0", 5 | "copyright":"Copyright 2020 Google LLC", 6 | "open":true, 7 | "icon":"arrow_forward", 8 | "title":"Storage To Table", 9 | "description":"Move using bucket and path prefix.", 10 | "from":["storage"], 11 | "to":["bigquery"], 12 | "instructions":[ 13 | "Specify a bucket and path prefix, * suffix is NOT required.", 14 | "Every time the job runs it will overwrite the table." 15 | ], 16 | "authors":["kenjora@google.com"] 17 | }, 18 | "tasks":[ 19 | { "bigquery":{ 20 | "auth":{"field":{ "name":"auth_read", "kind":"authentication", "order":1, "default":"user", "description":"Credentials used for reading data." }}, 21 | "from":{ 22 | "bucket":{"field":{ "name":"bucket", "kind":"string", "order":1, "default":"", "description":"Google cloud bucket." }}, 23 | "path":{"field":{ "name":"path", "kind":"string", "order":2, "default":"", "description":"Path prefix to read from, no * required." }} 24 | }, 25 | "to":{ 26 | "auth":{"field":{ "name":"auth_write", "kind":"authentication", "order":1, "default":"service", "description":"Credentials used for writing data." }}, 27 | "dataset":{"field":{ "name":"dataset", "kind":"string", "order":3, "default":"", "description":"Existing BigQuery dataset." }}, 28 | "table":{"field":{ "name":"table", "kind":"string", "order":4, "default":"", "description":"Table to create from this query." }} 29 | }, 30 | "schema":{"field":{ "name":"schema", "kind":"json", "order":5, "default":"[]", "description":"Schema provided in JSON list format or empty list." }} 31 | }} 32 | ] 33 | } 34 | -------------------------------------------------------------------------------- /tests/salesforce.json: -------------------------------------------------------------------------------- 1 | { 2 | "setup":{ 3 | "license":"Licensed under the Apache License, Version 2.0", 4 | "copyright":"Copyright 2020 Google LLC" 5 | }, 6 | "tasks":[ 7 | { "dataset":{ 8 | "auth":"user", 9 | "dataset":"Test" 10 | }}, 11 | { "include":{ 12 | "script":"scripts/salesforce_to_bigquery.json", 13 | "parameters":{ 14 | "domain":{"field":{ "name":"domain", "kind":"string", "description":"Retrieve from a Salesforce Domain." }}, 15 | "client":{"field":{ "name":"client", "kind":"string", "description":"Retrieve from a Salesforce App." }}, 16 | "secret":{"field":{ "name":"secret", "kind":"string", "description":"Retrieve from a Salesforce App." }}, 17 | "username":{"field":{ "name":"username", "kind":"email", "description":"Your Salesforce user email." }}, 18 | "password":{"field":{ "name":"password", "kind":"password", "description":"Your Salesforce login password." }}, 19 | "query":"SELECT Id, Name, Email FROM Contact", 20 | "dataset":"Test", 21 | "table":"Salesforce_Contact", 22 | "schema":[ 23 | { "name":"ID", "type":"STRING", "mode": "REQUIRED" }, 24 | { "name":"Name", "type":"STRING", "mode": "NULLABLE" }, 25 | { "name":"Email", "type":"STRING", "mode": "NULLABLE" } 26 | ] 27 | } 28 | }}, 29 | { "test": { 30 | "auth":"user", 31 | "bigquery":{ 32 | "dataset":"Test", 33 | "table":"Salesforce_Contact", 34 | "schema":[ 35 | { "name":"ID", "type":"STRING", "mode": "REQUIRED" }, 36 | { "name":"Name", "type":"STRING", "mode": "NULLABLE" }, 37 | { "name":"Email", "type":"STRING", "mode": "NULLABLE" } 38 | ] 39 | } 40 | }} 41 | ] 42 | } 43 | -------------------------------------------------------------------------------- /tutorials/cloud_client_web.md: -------------------------------------------------------------------------------- 1 | # StarThinker UI Client Credentials 2 | 3 | ## Create Client Credentials ( Web ) 4 | 5 | Retrieve oAuth Client Id from: https://console.cloud.google.com/apis/credentials 6 | 7 | ![Cloud Client UI](images/cloud_client_web.png) 8 | [View Cloud Client UI Setup Larger](images/cloud_client_web.png) 9 | 10 | ### Setup Steps 11 | 12 | 1. Google Cloud Project -> APIs & Services -> Credentials -> Create Credentials -> oAuth Client Id. 13 | 1. Select Web. 14 | 1. Name is StarThinker. 15 | 1. Auth Redirect URI is ( you can have several or change them later ): 16 | - http://localhost:8000/oauth_callback/ ( if you deploy the local UI ) 17 | - http://[your appengine instance ]/oauth_callback/ ( if you deploy on appengine ) 18 | - http://[your custom domain ]/oauth_callback/ ( if you deploy on a custom domain ) 19 | 1. Click the download icon next to the new credentials and open the downloaded file. 20 | 1. Copy the contents and paste into the StarThinker deployment script when prompted for Client Web Credentials. 21 | 22 | ### Change Steps 23 | 24 | If you change the Auth Redirect URI you will need to re-download the credentials into StarThinker. 25 | 26 | 1. Edit the credentials and add a new Auth Redirect URI. 27 | 1. Click the download icon next to the new credentials and open the downloaded file. 28 | 1. Copy the contents and paste into the StarThinker deployment script when prompted for Client Web Credentials. 29 | 1. ```source install/deploy.sh``` 30 | 1. ```7) Change UI Credentials``` 31 | 32 | ### Verify 33 | 34 | If all goes well, after deployment, the following file should exist with your credentials inside: 35 | 36 | - starthinker_assets/client_web.json 37 | 38 | --- 39 | © 2019 Google Inc. - Apache License, Version 2.0 40 | -------------------------------------------------------------------------------- /scripts/trends_places_to_bigquery_via_value.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2018-06-06", 4 | "license":"Licensed under the Apache License, Version 2.0", 5 | "copyright":"Copyright 2020 Google LLC", 6 | "ui":false, 7 | "icon":"perm_phone_msg", 8 | "title":"Trends Places To BigQuery Via Values", 9 | "description":"Move using hard coded WOEID values.", 10 | "from":["twitter"], 11 | "to":["bigquery"], 12 | "instructions":[ 13 | "Provide [Twitter Credentials](https://apps.twitter.com).", 14 | "Provide a comma delimited list of WOEIDs.", 15 | "Specify BigQuery dataset and table to write API call results to.", 16 | "Writes: WOEID, Name, Url, Promoted_Content, Query, Tweet_Volume", 17 | "Note Twitter API is rate limited to 15 requests per 15 minutes. So keep WOEID lists short." 18 | ], 19 | "authors":["kenjora@google.com"] 20 | }, 21 | "tasks":[ 22 | { "twitter":{ 23 | "auth":{"field":{ "name":"auth_write", "kind":"authentication", "order":1, "default":"service", "description":"Credentials used for writing data." }}, 24 | "secret":{"field":{ "name":"secret", "kind":"string", "order":1, "default":"" }}, 25 | "key":{"field":{ "name":"key", "kind":"string", "order":2, "default":"" }}, 26 | "trends":{ 27 | "places":{ 28 | "single_cell":true, 29 | "values": {"field":{ "name":"woeids", "kind":"integer_list", "order":3, "default":[] }} 30 | } 31 | }, 32 | "out":{ 33 | "bigquery":{ 34 | "dataset":{"field":{ "name":"destination_dataset", "kind":"string", "order":6, "default":"" }}, 35 | "table":{"field":{ "name":"destination_table", "kind":"string", "order":7, "default":"" }} 36 | } 37 | } 38 | }} 39 | ] 40 | } 41 | -------------------------------------------------------------------------------- /starthinker/util/salesforce.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | 19 | import json 20 | from urllib import request, parse 21 | 22 | from simple_salesforce import Salesforce 23 | 24 | 25 | def authenticate(domain, client_id, client_secret, username, password): 26 | 27 | data = parse.urlencode({ 28 | 'grant_type': 'password', 29 | 'client_id': client_id, 30 | 'client_secret': client_secret, 31 | 'username': username, 32 | 'password': password 33 | }).encode() 34 | 35 | req = request.Request('https://%s/services/oauth2/token' % domain, data=data) 36 | creds = json.loads(request.urlopen(req).read()) 37 | return Salesforce( 38 | instance_url=creds['instance_url'], session_id=creds['access_token']) 39 | 40 | 41 | def query(service, query, header=False): 42 | for record in service.query_all(query)['records']: 43 | if header: 44 | yield [column for column in record.keys() if column != 'attributes'] 45 | header = False 46 | yield [record[column] for column in record.keys() if column != 'attributes'] 47 | -------------------------------------------------------------------------------- /scripts/cm_campaign_audit.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2020-03-23", 4 | "license":"Licensed under the Apache License, Version 2.0", 5 | "copyright":"Copyright 2020 Google LLC", 6 | "icon":"done_outline", 7 | "title":"CM360 Campaign Auditor", 8 | "description":"A tool for rapidly bulk checking Campaign Manager campaigns", 9 | "image":"https://github.com/google/starthinker/raw/master/tutorials/images/cm_user_edit.png", 10 | "sample":"https://docs.google.com/spreadsheets/d/1tt597dMsAaxYXaJdifwKYNVzJrIl6E9Pe8GysfVrWOs/", 11 | "documentation":"https://docs.google.com/spreadsheets/d/1tt597dMsAaxYXaJdifwKYNVzJrIl6E9Pe8GysfVrWOs/", 12 | "from":["cm360", "apps script"], 13 | "to":["sheets"], 14 | "pitches":[ 15 | "View multiple campaigns at once.", 16 | "Write custom filters to find outliers.", 17 | "Rapidly audit entire campaign set for an account." 18 | ], 19 | "impacts":{ 20 | "spend optimization":0, 21 | "spend growth":0, 22 | "time savings":100, 23 | "account health":100, 24 | "csat improvement":100 25 | }, 26 | "instructions":[ 27 | "See all [Bulkdozer Lite](https://github.com/google/bulkdozer-lite#installation) related tools and instructions." 28 | ], 29 | "authors":["mauriciod@google.com"] 30 | }, 31 | "setup":{ 32 | "day":[], 33 | "hour":[] 34 | }, 35 | "tasks":[ 36 | { "drive": { 37 | "auth": "user", 38 | "hour":[], 39 | "copy":{ 40 | "source":"https://docs.google.com/spreadsheets/d/1tt597dMsAaxYXaJdifwKYNVzJrIl6E9Pe8GysfVrWOs/", 41 | "destination":{"field":{ "name":"recipe_name", "prefix":"CM User Editor For ", "kind":"string", "order":1, "description":"Name of document to deploy to.", "default":"" }} 42 | } 43 | }} 44 | ] 45 | } 46 | -------------------------------------------------------------------------------- /scripts/sheets_to_bigquery.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2017-12-05", 4 | "license":"Licensed under the Apache License, Version 2.0", 5 | "copyright":"Copyright 2020 Google LLC", 6 | "open":true, 7 | "icon":"view_column", 8 | "title":"Sheet To BigQuery", 9 | "description":"Import data from a sheet and move it to a BigQuery table.", 10 | "test":"https://github.com/google/starthinker/blob/master/tests/sheets.json", 11 | "from":["sheets"], 12 | "to":["bigquery"], 13 | "instructions":[ 14 | "For the sheet, provide the full edit URL.", 15 | "If the tab does not exist it will be created.", 16 | "Empty cells in the range will be NULL.", 17 | "Check Sheets header if first row is a header" 18 | ], 19 | "authors":["mauriciod@google.com"] 20 | }, 21 | "tasks":[ 22 | { "sheets":{ 23 | "auth":{"field":{ "name":"auth_read", "kind":"authentication", "order":0, "default":"user", "description":"Credentials used for reading data." }}, 24 | "sheet":{"field":{ "name":"sheets_url", "kind":"string", "order":2, "default":"" }}, 25 | "tab":{"field":{ "name":"sheets_tab", "kind":"string", "order":3, "default":"" }}, 26 | "range":{"field":{ "name":"sheets_range", "kind":"string", "order":4, "default":"" }}, 27 | "header":{"field":{ "name":"sheets_header", "kind":"boolean", "order":9, "default":true }}, 28 | "out":{ 29 | "auth":{"field":{ "name":"auth_write", "kind":"authentication", "order":1, "default":"service", "description":"Credentials used for writing data." }}, 30 | "bigquery":{ 31 | "dataset":{"field":{ "name":"dataset", "kind":"string", "order":5, "default":"" }}, 32 | "table":{"field":{ "name":"table", "kind":"string", "order":6, "default":"" }} 33 | } 34 | } 35 | }} 36 | ] 37 | } 38 | -------------------------------------------------------------------------------- /starthinker_ui/website/views.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | 19 | from django.shortcuts import render 20 | from django.template.loader import render_to_string 21 | 22 | from starthinker_ui.recipe.scripts import Script 23 | 24 | 25 | def help(request): 26 | # if open source then request will be null 27 | if request: 28 | return render(request, 'website/help.html', {}) 29 | else: 30 | return render_to_string('website/help.html', {'external': True}) 31 | 32 | 33 | def solutions(request): 34 | # if open source then request will be null 35 | if request: 36 | return render(request, 'website/solutions.html', {}) 37 | else: 38 | return render_to_string('website/solutions.html', {'external': True}) 39 | 40 | 41 | def solution(request, tag): 42 | script = Script(tag) 43 | # if open source then request will be null 44 | if request: 45 | return render(request, 'website/solution.html', {'script': script}) 46 | else: 47 | return render_to_string('website/solution.html', { 48 | 'script': script, 49 | 'external': True 50 | }) 51 | -------------------------------------------------------------------------------- /starthinker_ui/website/management/commands/airflow.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | 19 | from django.core.management.base import BaseCommand, CommandError 20 | from django.conf import settings 21 | 22 | from starthinker_ui.recipe.scripts import Script 23 | from starthinker_ui.recipe.dag import script_to_dag 24 | 25 | 26 | class Command(BaseCommand): 27 | help = 'Generate Templates For Airflow' 28 | 29 | def handle(self, *args, **kwargs): 30 | for script in Script.get_scripts(): 31 | if script.get_open_source(): 32 | print('Writing: %s_dag.py' % script.get_tag()) 33 | with open( 34 | '%s/dags/%s_dag.py' % ( 35 | settings.UI_ROOT, 36 | script.get_tag() 37 | ), 38 | 'w' 39 | ) as dag_file: 40 | dag_file.write( 41 | script_to_dag( 42 | script.get_tag(), 43 | script.get_name(), 44 | script.get_description(), 45 | script.get_instructions(), 46 | script.get_tasks_and_setup() 47 | ) 48 | ) 49 | -------------------------------------------------------------------------------- /scripts/kv_uploader.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2019-12-04", 4 | "license":"Licensed under the Apache License, Version 2.0", 5 | "copyright":"Copyright 2020 Google LLC", 6 | "icon":"policy", 7 | "title":"Tag Key Value Uploader", 8 | "description":"A tool for bulk editing key value pairs for CM placements.", 9 | "image":"https://github.com/google/starthinker/raw/master/tutorials/images/kv_uploader.png", 10 | "sample":"https://docs.google.com/spreadsheets/d/19Sxy4BDtK9ocq_INKTiZ-rZHgqhfpiiokXOTsYzmah0/edit?usp=sharing", 11 | "documentation":"https://github.com/google/starthinker/blob/master/tutorials/appscript_troubleshooting.md", 12 | "from":["apps script", "sheets"], 13 | "to":["cm360"], 14 | "pitches":[ 15 | "Update CM Placement key value pairs in bulk.", 16 | "Speed up tag editing.", 17 | "Ensure more consistent tagging." 18 | ], 19 | "impacts":{ 20 | "spend optimization":0, 21 | "spend growth":20, 22 | "time savings":100, 23 | "account health":50, 24 | "csat improvement":80 25 | }, 26 | "instructions":[ 27 | "Add this card to a recipe and save it.", 28 | "Then click **Run Now** to deploy.", 29 | "Follow the instructions in the sheet for setup." 30 | ], 31 | "authors":["kenjora@google.com"] 32 | }, 33 | "setup":{ 34 | "day":[], 35 | "hour":[] 36 | }, 37 | "tasks":[ 38 | { "drive": { 39 | "auth": "user", 40 | "hour":[], 41 | "copy":{ 42 | "source":"https://docs.google.com/spreadsheets/d/19Sxy4BDtK9ocq_INKTiZ-rZHgqhfpiiokXOTsYzmah0/", 43 | "destination":{"field":{ "name":"recipe_name", "prefix":"Key Value Uploader For ", "kind":"string", "order":1, "description":"Name of document to deploy to.", "default":"" }} 44 | } 45 | }} 46 | ] 47 | } 48 | -------------------------------------------------------------------------------- /starthinker_ui/account/decorators.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | 19 | import json 20 | from functools import wraps 21 | 22 | from django.conf import settings 23 | from django.http import HttpResponseRedirect 24 | 25 | from starthinker.util.auth_wrapper import CredentialsFlowWrapper 26 | 27 | 28 | def permission_admin(): 29 | 30 | def _decorator(_view): 31 | 32 | @wraps(_view) 33 | def _wrapper(request, *args, **kwargs): 34 | 35 | # user is logged in 36 | if request.user.is_authenticated: 37 | return _view(request, *args, **kwargs) 38 | 39 | # multi user mode, log user in using oauth 40 | else: 41 | flow = CredentialsFlowWrapper( 42 | settings.UI_CLIENT, 43 | redirect_uri=settings.CONST_URL + '/oauth_callback/') 44 | auth_url, _ = flow.authorization_url( 45 | prompt='', access_type='offline', include_granted_scopes='true') 46 | request.session['code_verifier'] = flow.code_verifier 47 | return HttpResponseRedirect(auth_url) 48 | 49 | return _wrapper 50 | 51 | return _decorator 52 | -------------------------------------------------------------------------------- /scripts/interaction_analyzer.json: -------------------------------------------------------------------------------- 1 | { 2 | "script":{ 3 | "released":"2020-05-26", 4 | "icon":"security", 5 | "title":"Interaction Analyzer", 6 | "description":"A tool for viewing calendar and email interactions for tracking work quicker.", 7 | "image":"https://drive.google.com/file/d/1KBeZkv0gncjNV1dvg6IUkcF-SPPgGm9F/view?usp=sharing", 8 | "sample":"https://docs.google.com/spreadsheets/d/164ey9QUvu6J4qxAGd21FUGFZTtjmXPgES9UHGu2CQ5A/", 9 | "from":["gmail", "calendar", "plx", "sheets", "apps script"], 10 | "to":["datastudio"], 11 | "pitches":[ 12 | "View distribution of calendar meetings.", 13 | "View distribution of email threads.", 14 | "Find interactions with specific clients.", 15 | "Check whom each interaction rolls up to.", 16 | "AI identifies possible interactions to track." 17 | ], 18 | "impacts":{ 19 | "spend optimization":0, 20 | "spend growth":0, 21 | "time savings":100, 22 | "account health":100, 23 | "csat improvement":100 24 | }, 25 | "instructions":[ 26 | "Add this card to a recipe and save it.", 27 | "Then click **Run Now** to deploy.", 28 | "Follow the [instructions](https://docs.google.com/spreadsheets/d/164ey9QUvu6J4qxAGd21FUGFZTtjmXPgES9UHGu2CQ5A/) for setup." 29 | ], 30 | "authors":["kenjora@google.com"] 31 | }, 32 | "setup":{ 33 | "day":[], 34 | "hour":[] 35 | }, 36 | "tasks":[ 37 | { "drive": { 38 | "auth": "user", 39 | "hour":[], 40 | "copy":{ 41 | "source":"https://docs.google.com/spreadsheets/d/164ey9QUvu6J4qxAGd21FUGFZTtjmXPgES9UHGu2CQ5A/", 42 | "destination":{"field":{ "name":"recipe_name", "prefix":"CM User Editor For ", "kind":"string", "order":1, "description":"Name of document to deploy to.", "default":"" }} 43 | } 44 | }} 45 | ] 46 | } 47 | -------------------------------------------------------------------------------- /tests/ctv_audience_affinity.json: -------------------------------------------------------------------------------- 1 | { 2 | "setup": { 3 | "license":"Licensed under the Apache License, Version 2.0", 4 | "copyright":"Copyright 2020 Google LLC", 5 | "hour": [ 3 ], 6 | "day": [ "Mon" ] 7 | }, 8 | "tasks": [ 9 | { "include":{ 10 | "script":"scripts/ctv_audience_affinity.json", 11 | "parameters":{ 12 | "recipe_name":"ctv_test", 13 | "partner_id":234340, 14 | "dataset":"ctv_test", 15 | "audience_ids":[547327893,728185126,614644370,616756026,484169638] 16 | } 17 | }}, 18 | { "test": { 19 | "auth":"user", 20 | "bigquery":{ 21 | "dataset":"ctv_test", 22 | "table":"final_table", 23 | "schema": [ 24 | {"mode": "NULLABLE", "name": "app_url", "type": "STRING"}, 25 | {"mode": "NULLABLE", "name": "ctv_app_name", "type": "STRING"}, 26 | {"mode": "NULLABLE", "name": "app_or_domain", "type": "STRING"}, 27 | {"mode": "NULLABLE", "name": "audience_list", "type": "STRING"}, 28 | {"mode": "NULLABLE", "name": "audience_app_impressions", "type": "INTEGER"}, 29 | {"mode": "NULLABLE", "name": "audience_app_uniques", "type": "INTEGER"}, 30 | {"mode": "NULLABLE", "name": "audience_baseline_impressions", "type": "INTEGER"}, 31 | {"mode": "NULLABLE", "name": "audience_baseline_uniques", "type": "INTEGER"}, 32 | {"mode": "NULLABLE", "name": "country_app_impressions", "type": "INTEGER"}, 33 | {"mode": "NULLABLE", "name": "country_app_uniques", "type": "INTEGER"}, 34 | {"mode": "NULLABLE", "name": "country_baseline_impressions", "type": "INTEGER"}, 35 | {"mode": "NULLABLE", "name": "country_baseline_uniques", "type": "INTEGER"}, 36 | {"mode": "NULLABLE", "name": "affinity_index", "type": "FLOAT"} 37 | ] 38 | } 39 | }} 40 | ] 41 | } 42 | -------------------------------------------------------------------------------- /tutorials/testing_ui.md: -------------------------------------------------------------------------------- 1 | # UI Tests 2 | 3 | UI tests use the open source out of the box [Django Test Framework](https://docs.djangoproject.com/en/2.2/topics/testing/). 4 | These are standard Python unit tests with a few additional Django helpers. Tests use command line credentials. Tests can 5 | be executed per Django instructons or via the helper bash script: 6 | 7 | ## Steps 8 | ``` 9 | bash install/deploy.sh 10 | ``` 11 | 12 | 1. Option 1) Developer Menu 13 | 1. Option 3) Test UI 14 | - You may be asked for a Cloud Project ID ( use the ID, not the Name, not the Number ) 15 | - You may be asked for [Service Credentials](cloud_service.md). 16 | - You may be asked for [Installed Client Credentials](cloud_client_installed.md). 17 | 18 | ## Code 19 | 20 | - [starthinker_ui/recipe/tests.py](../starthinker_ui/recipe/tests.py) 21 | - [starthinker_ui/website/tests.py](../starthinker_ui/website/tests.py) 22 | - [starthinker_ui/account/tests.py](../starthinker_ui/account/tests.py) 23 | - [starthinker_ui/project/tests.py](../starthinker_ui/project/tests.py) 24 | 25 | ## Manual Testing 26 | 27 | When the UI is active it saves recipes to the local database. Those recipes can be executed by the worker 28 | using the following commands: 29 | 30 | ``` 31 | source starthinker_assets/development.sh 32 | python starthinker_ui/manage.py job_worker --test --verbose --jobs 1 33 | ``` 34 | 35 | This will execute one loop of the worker, pulling one recipe at a time from the local databse and executing 36 | it as the worker. 37 | 38 | 39 | ## Cloud Resources 40 | 41 | - [Google Cloud Credentials](https://console.cloud.google.com/apis/credentials) - where you manage your credentials. 42 | - [Google Cloud Billing](https://console.cloud.google.com/billing/linkedaccount) - examine costs in real time. 43 | 44 | --- 45 | © 2019 Google Inc. - Apache License, Version 2.0 46 | -------------------------------------------------------------------------------- /starthinker_ui/website/management/commands/example.py: -------------------------------------------------------------------------------- 1 | ########################################################################### 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ########################################################################### 18 | 19 | from django.core.management.base import BaseCommand, CommandError 20 | from django.conf import settings 21 | 22 | from starthinker_ui.recipe.scripts import Script 23 | from starthinker.tool.example import recipe_to_python 24 | 25 | 26 | class Command(BaseCommand): 27 | help = 'Generate Templates For Python' 28 | 29 | def handle(self, *args, **kwargs): 30 | for script in Script.get_scripts(): 31 | if script.get_tag() in ['airflow']: continue 32 | if script.get_open_source(): 33 | print('Writing: %s_example.py' % script.get_tag()) 34 | with open( 35 | '%s/examples/%s_example.py' % ( 36 | settings.UI_ROOT, 37 | script.get_tag() 38 | ), 39 | 'w' 40 | ) as py_file: 41 | py_file.write( 42 | recipe_to_python( 43 | script.get_tag(), 44 | script.get_description(), 45 | script.get_instructions(), 46 | script.get_tasks() 47 | ) 48 | ) 49 | --------------------------------------------------------------------------------