├── .gitignore ├── .travis.yml ├── ANNOTATION.md ├── Dockerfile ├── LICENSE ├── README.md ├── annotation └── exports │ ├── tcpinfo_annotation_export.sql │ └── traceroute_hopannotation1_export.sql ├── cloudbuild.yaml ├── cmd └── stats-pipeline │ └── main.go ├── compose-annotation-export.yaml ├── compose-hopannotation1-export.yaml ├── config.json ├── config └── config.go ├── cors-settings.json ├── create_statistics_api.sh ├── docs ├── api-structure.md ├── format-schema.md ├── geo-precision.md └── stats-overview.md ├── exporter ├── exporter.go ├── exporter_test.go └── testdata │ └── export_query.sql ├── formatter ├── annotation.go ├── annotation_test.go ├── hopannotation1.go ├── hopannotation1_test.go ├── stats.go └── stats_test.go ├── go.mod ├── go.sum ├── histogram ├── table.go └── table_test.go ├── k8s └── data-pipeline │ ├── config │ ├── config-annotation-export.json │ ├── config-hopannotation1-export.json │ └── config.json │ ├── deployments │ ├── hopannotation1-export-template.yaml │ └── stats-pipeline.yaml.template │ ├── jobs │ ├── hopannotation1-export-cronjob.template │ └── stats-pipeline-cronjob.yaml.template │ └── services │ ├── hopannotation1-export.yaml │ └── stats-pipeline.yaml ├── maptiles ├── Dockerfile ├── Makefile ├── README.md ├── package-lock.json ├── package.json ├── run-pipeline.sh └── scripts │ ├── combine-blocks.sh │ ├── download-blocks.js │ ├── download-mlab.js │ ├── load-fcc-477-to-db.js │ ├── process-fcc.js │ ├── process-mlab.js │ └── unzip-blocks.sh ├── output ├── writer.go └── writer_test.go ├── pipeline ├── handlers.go ├── handlers_test.go ├── headers.go └── testdata │ ├── test_export.sql │ └── test_histogram.sql └── statistics ├── exports ├── cities.sql ├── cities_asn.sql ├── continents.sql ├── continents_asn.sql ├── countries.sql ├── countries_asn.sql ├── global_asn.sql ├── regions.sql ├── regions_asn.sql ├── us_counties.sql ├── us_counties_asn.sql ├── us_states.sql ├── us_states_asn.sql ├── us_tracts.sql └── us_tracts_asn.sql ├── queries ├── canary.sql ├── continent_asn_histogram.sql ├── continent_country_asn_histogram.sql ├── continent_country_histogram.sql ├── continent_country_region_asn_histogram.sql ├── continent_country_region_city_asn_histogram.sql ├── continent_country_region_city_histogram.sql ├── continent_country_region_histogram.sql ├── continent_histogram.sql ├── global_asn_histogram.sql ├── us_census_tracts_asn_histogram.sql ├── us_census_tracts_histogram.sql ├── us_county_asn_histogram.sql ├── us_county_histogram.sql ├── us_state_territories_asn_histogram.sql └── us_state_territories_histogram.sql └── scripts └── update_stats_continent_country_region_histogram.sh /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode/ 2 | stats-pipeline 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/.travis.yml -------------------------------------------------------------------------------- /ANNOTATION.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/ANNOTATION.md -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/Dockerfile -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/README.md -------------------------------------------------------------------------------- /annotation/exports/tcpinfo_annotation_export.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/annotation/exports/tcpinfo_annotation_export.sql -------------------------------------------------------------------------------- /annotation/exports/traceroute_hopannotation1_export.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/annotation/exports/traceroute_hopannotation1_export.sql -------------------------------------------------------------------------------- /cloudbuild.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/cloudbuild.yaml -------------------------------------------------------------------------------- /cmd/stats-pipeline/main.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/cmd/stats-pipeline/main.go -------------------------------------------------------------------------------- /compose-annotation-export.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/compose-annotation-export.yaml -------------------------------------------------------------------------------- /compose-hopannotation1-export.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/compose-hopannotation1-export.yaml -------------------------------------------------------------------------------- /config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/config.json -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/config/config.go -------------------------------------------------------------------------------- /cors-settings.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/cors-settings.json -------------------------------------------------------------------------------- /create_statistics_api.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/create_statistics_api.sh -------------------------------------------------------------------------------- /docs/api-structure.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/docs/api-structure.md -------------------------------------------------------------------------------- /docs/format-schema.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/docs/format-schema.md -------------------------------------------------------------------------------- /docs/geo-precision.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/docs/geo-precision.md -------------------------------------------------------------------------------- /docs/stats-overview.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/docs/stats-overview.md -------------------------------------------------------------------------------- /exporter/exporter.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/exporter/exporter.go -------------------------------------------------------------------------------- /exporter/exporter_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/exporter/exporter_test.go -------------------------------------------------------------------------------- /exporter/testdata/export_query.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/exporter/testdata/export_query.sql -------------------------------------------------------------------------------- /formatter/annotation.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/formatter/annotation.go -------------------------------------------------------------------------------- /formatter/annotation_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/formatter/annotation_test.go -------------------------------------------------------------------------------- /formatter/hopannotation1.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/formatter/hopannotation1.go -------------------------------------------------------------------------------- /formatter/hopannotation1_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/formatter/hopannotation1_test.go -------------------------------------------------------------------------------- /formatter/stats.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/formatter/stats.go -------------------------------------------------------------------------------- /formatter/stats_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/formatter/stats_test.go -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/go.mod -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/go.sum -------------------------------------------------------------------------------- /histogram/table.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/histogram/table.go -------------------------------------------------------------------------------- /histogram/table_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/histogram/table_test.go -------------------------------------------------------------------------------- /k8s/data-pipeline/config/config-annotation-export.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/k8s/data-pipeline/config/config-annotation-export.json -------------------------------------------------------------------------------- /k8s/data-pipeline/config/config-hopannotation1-export.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/k8s/data-pipeline/config/config-hopannotation1-export.json -------------------------------------------------------------------------------- /k8s/data-pipeline/config/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/k8s/data-pipeline/config/config.json -------------------------------------------------------------------------------- /k8s/data-pipeline/deployments/hopannotation1-export-template.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/k8s/data-pipeline/deployments/hopannotation1-export-template.yaml -------------------------------------------------------------------------------- /k8s/data-pipeline/deployments/stats-pipeline.yaml.template: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/k8s/data-pipeline/deployments/stats-pipeline.yaml.template -------------------------------------------------------------------------------- /k8s/data-pipeline/jobs/hopannotation1-export-cronjob.template: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/k8s/data-pipeline/jobs/hopannotation1-export-cronjob.template -------------------------------------------------------------------------------- /k8s/data-pipeline/jobs/stats-pipeline-cronjob.yaml.template: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/k8s/data-pipeline/jobs/stats-pipeline-cronjob.yaml.template -------------------------------------------------------------------------------- /k8s/data-pipeline/services/hopannotation1-export.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/k8s/data-pipeline/services/hopannotation1-export.yaml -------------------------------------------------------------------------------- /k8s/data-pipeline/services/stats-pipeline.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/k8s/data-pipeline/services/stats-pipeline.yaml -------------------------------------------------------------------------------- /maptiles/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/maptiles/Dockerfile -------------------------------------------------------------------------------- /maptiles/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/maptiles/Makefile -------------------------------------------------------------------------------- /maptiles/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/maptiles/README.md -------------------------------------------------------------------------------- /maptiles/package-lock.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/maptiles/package-lock.json -------------------------------------------------------------------------------- /maptiles/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/maptiles/package.json -------------------------------------------------------------------------------- /maptiles/run-pipeline.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/maptiles/run-pipeline.sh -------------------------------------------------------------------------------- /maptiles/scripts/combine-blocks.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/maptiles/scripts/combine-blocks.sh -------------------------------------------------------------------------------- /maptiles/scripts/download-blocks.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/maptiles/scripts/download-blocks.js -------------------------------------------------------------------------------- /maptiles/scripts/download-mlab.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/maptiles/scripts/download-mlab.js -------------------------------------------------------------------------------- /maptiles/scripts/load-fcc-477-to-db.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/maptiles/scripts/load-fcc-477-to-db.js -------------------------------------------------------------------------------- /maptiles/scripts/process-fcc.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/maptiles/scripts/process-fcc.js -------------------------------------------------------------------------------- /maptiles/scripts/process-mlab.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/maptiles/scripts/process-mlab.js -------------------------------------------------------------------------------- /maptiles/scripts/unzip-blocks.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/maptiles/scripts/unzip-blocks.sh -------------------------------------------------------------------------------- /output/writer.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/output/writer.go -------------------------------------------------------------------------------- /output/writer_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/output/writer_test.go -------------------------------------------------------------------------------- /pipeline/handlers.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/pipeline/handlers.go -------------------------------------------------------------------------------- /pipeline/handlers_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/pipeline/handlers_test.go -------------------------------------------------------------------------------- /pipeline/headers.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/pipeline/headers.go -------------------------------------------------------------------------------- /pipeline/testdata/test_export.sql: -------------------------------------------------------------------------------- 1 | SELECT * FROM {{.sourceTable}} -------------------------------------------------------------------------------- /pipeline/testdata/test_histogram.sql: -------------------------------------------------------------------------------- 1 | SELECT * FROM test -------------------------------------------------------------------------------- /statistics/exports/cities.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/exports/cities.sql -------------------------------------------------------------------------------- /statistics/exports/cities_asn.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/exports/cities_asn.sql -------------------------------------------------------------------------------- /statistics/exports/continents.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/exports/continents.sql -------------------------------------------------------------------------------- /statistics/exports/continents_asn.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/exports/continents_asn.sql -------------------------------------------------------------------------------- /statistics/exports/countries.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/exports/countries.sql -------------------------------------------------------------------------------- /statistics/exports/countries_asn.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/exports/countries_asn.sql -------------------------------------------------------------------------------- /statistics/exports/global_asn.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/exports/global_asn.sql -------------------------------------------------------------------------------- /statistics/exports/regions.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/exports/regions.sql -------------------------------------------------------------------------------- /statistics/exports/regions_asn.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/exports/regions_asn.sql -------------------------------------------------------------------------------- /statistics/exports/us_counties.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/exports/us_counties.sql -------------------------------------------------------------------------------- /statistics/exports/us_counties_asn.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/exports/us_counties_asn.sql -------------------------------------------------------------------------------- /statistics/exports/us_states.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/exports/us_states.sql -------------------------------------------------------------------------------- /statistics/exports/us_states_asn.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/exports/us_states_asn.sql -------------------------------------------------------------------------------- /statistics/exports/us_tracts.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/exports/us_tracts.sql -------------------------------------------------------------------------------- /statistics/exports/us_tracts_asn.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/exports/us_tracts_asn.sql -------------------------------------------------------------------------------- /statistics/queries/canary.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/queries/canary.sql -------------------------------------------------------------------------------- /statistics/queries/continent_asn_histogram.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/queries/continent_asn_histogram.sql -------------------------------------------------------------------------------- /statistics/queries/continent_country_asn_histogram.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/queries/continent_country_asn_histogram.sql -------------------------------------------------------------------------------- /statistics/queries/continent_country_histogram.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/queries/continent_country_histogram.sql -------------------------------------------------------------------------------- /statistics/queries/continent_country_region_asn_histogram.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/queries/continent_country_region_asn_histogram.sql -------------------------------------------------------------------------------- /statistics/queries/continent_country_region_city_asn_histogram.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/queries/continent_country_region_city_asn_histogram.sql -------------------------------------------------------------------------------- /statistics/queries/continent_country_region_city_histogram.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/queries/continent_country_region_city_histogram.sql -------------------------------------------------------------------------------- /statistics/queries/continent_country_region_histogram.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/queries/continent_country_region_histogram.sql -------------------------------------------------------------------------------- /statistics/queries/continent_histogram.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/queries/continent_histogram.sql -------------------------------------------------------------------------------- /statistics/queries/global_asn_histogram.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/queries/global_asn_histogram.sql -------------------------------------------------------------------------------- /statistics/queries/us_census_tracts_asn_histogram.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/queries/us_census_tracts_asn_histogram.sql -------------------------------------------------------------------------------- /statistics/queries/us_census_tracts_histogram.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/queries/us_census_tracts_histogram.sql -------------------------------------------------------------------------------- /statistics/queries/us_county_asn_histogram.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/queries/us_county_asn_histogram.sql -------------------------------------------------------------------------------- /statistics/queries/us_county_histogram.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/queries/us_county_histogram.sql -------------------------------------------------------------------------------- /statistics/queries/us_state_territories_asn_histogram.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/queries/us_state_territories_asn_histogram.sql -------------------------------------------------------------------------------- /statistics/queries/us_state_territories_histogram.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/queries/us_state_territories_histogram.sql -------------------------------------------------------------------------------- /statistics/scripts/update_stats_continent_country_region_histogram.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/m-lab/stats-pipeline/HEAD/statistics/scripts/update_stats_continent_country_region_histogram.sh --------------------------------------------------------------------------------