├── .dockerignore ├── bin ├── update ├── deploy-ci.sh ├── docker-update └── analytics ├── test ├── support │ ├── fixtures │ │ ├── secret_key.pem │ │ ├── secret_key.json │ │ ├── report.js │ │ ├── data.js │ │ ├── data_with_hostname.js │ │ └── results.js │ ├── mocks │ │ ├── googleapis-auth.js │ │ └── googleapis-analytics.js │ └── database.js ├── publish │ ├── disk.test.js │ ├── s3.test.js │ └── postgres.test.js ├── google-analytics │ ├── credential-loader.test.js │ ├── query-builder.test.js │ ├── client.test.js │ └── query-authorizer.test.js ├── analytics.test.js ├── process-results │ ├── result-formatter.test.js │ ├── ga-data-processor.test.js │ └── result-totals-calculator.test.js └── index.test.js ├── circle.yml ├── deploy ├── envs │ ├── labor.env │ ├── defense.env │ ├── justice.env │ ├── commerce.env │ ├── energy.env │ ├── state.env │ ├── education.env │ ├── interior.env │ ├── treasury.env │ ├── postal-service.env │ ├── agriculture.env │ ├── transportation.env │ ├── veterans-affairs.env │ ├── homeland-security.env │ ├── health-human-services.env │ ├── executive-office-president.env │ ├── national-science-foundation.env │ ├── office-personnel-management.env │ ├── nuclear-regulatory-commission.env │ ├── small-business-administration.env │ ├── housing-urban-development.env │ ├── social-security-administration.env │ ├── general-services-administration.env │ ├── agency-international-development.env │ ├── environmental-protection-agency.env │ ├── national-archives-records-administration.env │ └── national-aeronautics-space-administration.env ├── cron.js ├── api.sh ├── hourly.sh ├── daily.sh └── realtime.sh ├── newrelic.js ├── .gitignore ├── Dockerfile ├── knexfile.js ├── migrations ├── 20170308164751_create_analytics_data.js ├── 20170522094056_rename_date_time_to_date.js └── 20170316115145_add_analytics_data_indexes.js ├── src ├── publish │ ├── disk.js │ ├── s3.js │ └── postgres.js ├── google-analytics │ ├── credential-loader.js │ ├── query-builder.js │ ├── client.js │ └── query-authorizer.js ├── analytics.js ├── process-results │ ├── result-formatter.js │ ├── result-totals-calculator.js │ └── ga-data-processor.js └── config.js ├── env.example ├── manifest.yml ├── docker-compose.yml ├── CONTRIBUTING.md ├── LICENSE.md ├── package.json ├── index.js ├── reports ├── api.json ├── reports.json └── usa.json └── README.md /.dockerignore: -------------------------------------------------------------------------------- 1 | .gitignore -------------------------------------------------------------------------------- /bin/update: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | npm install 4 | npm run migrate 5 | -------------------------------------------------------------------------------- /test/support/fixtures/secret_key.pem: -------------------------------------------------------------------------------- 1 | pem-key-file-not-actually-a-secret-key 2 | -------------------------------------------------------------------------------- /circle.yml: -------------------------------------------------------------------------------- 1 | machine: 2 | node: 3 | version: 7 4 | deployment: 5 | master: 6 | branch: master 7 | commands: 8 | - bin/deploy-ci.sh 9 | -------------------------------------------------------------------------------- /deploy/envs/labor.env: -------------------------------------------------------------------------------- 1 | # Department of Labor 2 | export ANALYTICS_REPORT_IDS="ga:66158666" 3 | export AGENCY_NAME=labor 4 | export AWS_BUCKET_PATH=data/labor 5 | -------------------------------------------------------------------------------- /deploy/envs/defense.env: -------------------------------------------------------------------------------- 1 | # Department of Defense 2 | export ANALYTICS_REPORT_IDS="ga:67120289" 3 | export AGENCY_NAME=defense 4 | export AWS_BUCKET_PATH=data/defense 5 | -------------------------------------------------------------------------------- /deploy/envs/justice.env: -------------------------------------------------------------------------------- 1 | # Department of Justice 2 | export ANALYTICS_REPORT_IDS="ga:65501425" 3 | export AGENCY_NAME=justice 4 | export AWS_BUCKET_PATH=data/justice 5 | -------------------------------------------------------------------------------- /deploy/envs/commerce.env: -------------------------------------------------------------------------------- 1 | # Department of Commerce 2 | export ANALYTICS_REPORT_IDS="ga:66877186" 3 | export AGENCY_NAME=commerce 4 | export AWS_BUCKET_PATH=data/commerce 5 | -------------------------------------------------------------------------------- /deploy/envs/energy.env: -------------------------------------------------------------------------------- 1 | # Department of Energy 2 | export ANALYTICS_REPORT_IDS="ga:69826574" 3 | export AGENCY_NAME=energy 4 | export AWS_BUCKET_PATH=data/energy 5 | 6 | -------------------------------------------------------------------------------- /deploy/envs/state.env: -------------------------------------------------------------------------------- 1 | # Department of Transportation 2 | export ANALYTICS_REPORT_IDS="ga:67454734" 3 | export AGENCY_NAME=state 4 | export AWS_BUCKET_PATH=data/state 5 | -------------------------------------------------------------------------------- /deploy/envs/education.env: -------------------------------------------------------------------------------- 1 | # Department of Education 2 | export ANALYTICS_REPORT_IDS="ga:67200736" 3 | export AGENCY_NAME=education 4 | export AWS_BUCKET_PATH=data/education 5 | -------------------------------------------------------------------------------- /deploy/envs/interior.env: -------------------------------------------------------------------------------- 1 | # Department of the Interior 2 | export ANALYTICS_REPORT_IDS="ga:65366693" 3 | export AGENCY_NAME=interior 4 | export AWS_BUCKET_PATH=data/interior 5 | -------------------------------------------------------------------------------- /deploy/envs/treasury.env: -------------------------------------------------------------------------------- 1 | # Department of the Treasury 2 | export ANALYTICS_REPORT_IDS="ga:67705218" 3 | export AGENCY_NAME=treasury 4 | export AWS_BUCKET_PATH=data/treasury 5 | -------------------------------------------------------------------------------- /deploy/envs/postal-service.env: -------------------------------------------------------------------------------- 1 | # Postal Service 2 | export ANALYTICS_REPORT_IDS="ga:100911348" 3 | export AGENCY_NAME=postal-service 4 | export AWS_BUCKET_PATH=data/postal-service 5 | -------------------------------------------------------------------------------- /deploy/envs/agriculture.env: -------------------------------------------------------------------------------- 1 | # Department of Agriculture 2 | export ANALYTICS_REPORT_IDS="ga:65240995" 3 | export AGENCY_NAME=agriculture 4 | export AWS_BUCKET_PATH=data/agriculture 5 | 6 | -------------------------------------------------------------------------------- /newrelic.js: -------------------------------------------------------------------------------- 1 | exports.config = { 2 | app_name: [process.env.NEW_RELIC_APP_NAME], 3 | license_key: process.env.NEW_RELIC_LICENSE_KEY, 4 | logging: { 5 | level: "info" 6 | }, 7 | } 8 | -------------------------------------------------------------------------------- /deploy/envs/transportation.env: -------------------------------------------------------------------------------- 1 | # Department of Transportation 2 | export ANALYTICS_REPORT_IDS="ga:66902419" 3 | export AGENCY_NAME=transportation 4 | export AWS_BUCKET_PATH=data/transportation 5 | -------------------------------------------------------------------------------- /deploy/envs/veterans-affairs.env: -------------------------------------------------------------------------------- 1 | # Department of Veterans Affairs 2 | export ANALYTICS_REPORT_IDS="ga:68076838" 3 | export AGENCY_NAME=veterans-affairs 4 | export AWS_BUCKET_PATH=data/veterans-affairs 5 | -------------------------------------------------------------------------------- /deploy/envs/homeland-security.env: -------------------------------------------------------------------------------- 1 | # Department of Homeland Security 2 | export ANALYTICS_REPORT_IDS="ga:67460690" 3 | export AGENCY_NAME=homeland-security 4 | export AWS_BUCKET_PATH=data/homeland-security 5 | 6 | -------------------------------------------------------------------------------- /deploy/envs/health-human-services.env: -------------------------------------------------------------------------------- 1 | # Department of Health and Human Services 2 | export ANALYTICS_REPORT_IDS="ga:72643802" 3 | export AGENCY_NAME=health-human-services 4 | export AWS_BUCKET_PATH=data/health-human-services 5 | 6 | -------------------------------------------------------------------------------- /deploy/envs/executive-office-president.env: -------------------------------------------------------------------------------- 1 | # Executive Office of the President 2 | export ANALYTICS_REPORT_IDS="ga:66351974" 3 | export AGENCY_NAME=executive-office-president 4 | export AWS_BUCKET_PATH=data/executive-office-president 5 | -------------------------------------------------------------------------------- /deploy/envs/national-science-foundation.env: -------------------------------------------------------------------------------- 1 | # National Science Foundation 2 | export ANALYTICS_REPORT_IDS="ga:67850613" 3 | export AGENCY_NAME=national-science-foundation 4 | export AWS_BUCKET_PATH=data/national-science-foundation 5 | -------------------------------------------------------------------------------- /deploy/envs/office-personnel-management.env: -------------------------------------------------------------------------------- 1 | # Office of Personnel Management 2 | export ANALYTICS_REPORT_IDS="ga:68758375" 3 | export AGENCY_NAME=office-personnel-management 4 | export AWS_BUCKET_PATH=data/office-personnel-management 5 | -------------------------------------------------------------------------------- /deploy/envs/nuclear-regulatory-commission.env: -------------------------------------------------------------------------------- 1 | # Nuclear Regulatory Commission 2 | export ANALYTICS_REPORT_IDS="ga:67691948" 3 | export AGENCY_NAME=nuclear-regulatory-commission 4 | export AWS_BUCKET_PATH=data/nuclear-regulatory-commission 5 | -------------------------------------------------------------------------------- /deploy/envs/small-business-administration.env: -------------------------------------------------------------------------------- 1 | # Small Business Administration 2 | export ANALYTICS_REPORT_IDS="ga:68909496" 3 | export AGENCY_NAME=small-business-administration 4 | export AWS_BUCKET_PATH=data/small-business-administration 5 | -------------------------------------------------------------------------------- /deploy/envs/housing-urban-development.env: -------------------------------------------------------------------------------- 1 | # Department of Housing and Urban Development 2 | export ANALYTICS_REPORT_IDS="ga:69364976" 3 | export AGENCY_NAME=housing-urban-development 4 | export AWS_BUCKET_PATH=data/housing-urban-development 5 | -------------------------------------------------------------------------------- /deploy/envs/social-security-administration.env: -------------------------------------------------------------------------------- 1 | # Social Security Administration 2 | export ANALYTICS_REPORT_IDS="ga:68055007" 3 | export AGENCY_NAME=social-security-administration 4 | export AWS_BUCKET_PATH=data/social-security-administration 5 | -------------------------------------------------------------------------------- /deploy/envs/general-services-administration.env: -------------------------------------------------------------------------------- 1 | # General Services Administration 2 | export ANALYTICS_REPORT_IDS="ga:65263002" 3 | export AGENCY_NAME=general-services-administration 4 | export AWS_BUCKET_PATH=data/general-services-administration 5 | -------------------------------------------------------------------------------- /deploy/envs/agency-international-development.env: -------------------------------------------------------------------------------- 1 | # Agency for International Development 2 | export ANALYTICS_REPORT_IDS="ga:68380943" 3 | export AGENCY_NAME=agency-international-development 4 | export AWS_BUCKET_PATH=data/agency-international-development 5 | -------------------------------------------------------------------------------- /deploy/envs/environmental-protection-agency.env: -------------------------------------------------------------------------------- 1 | # Environmental Protection Agency 2 | export ANALYTICS_REPORT_IDS="ga:68948437" 3 | export AGENCY_NAME=environmental-protection-agency 4 | export AWS_BUCKET_PATH=data/environmental-protection-agency 5 | 6 | -------------------------------------------------------------------------------- /test/support/fixtures/secret_key.json: -------------------------------------------------------------------------------- 1 | { 2 | "private_key_id": "123abc", 3 | "private_key": "json-key-file-not-actually-a-secret-key", 4 | "client_email": "json_test_email@example.com", 5 | "client_id": "789ghi.apps.googleusercontent.com", 6 | "type": "service_account" 7 | } 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.p12 3 | *.pem 4 | key_file.txt 5 | node_modules 6 | .python-version 7 | *.env 8 | *.json 9 | *.csv 10 | *.gz 11 | /data 12 | todo.txt 13 | npm-debug.log 14 | *.swp 15 | .git 16 | 17 | # Track selected JSON files 18 | !package.json 19 | !reports/*.json 20 | -------------------------------------------------------------------------------- /deploy/envs/national-archives-records-administration.env: -------------------------------------------------------------------------------- 1 | # National Archives and Records Administration 2 | export ANALYTICS_REPORT_IDS="ga:67886610" 3 | export AGENCY_NAME=national-archives-records-administration 4 | export AWS_BUCKET_PATH=data/national-archives-records-administration 5 | -------------------------------------------------------------------------------- /deploy/envs/national-aeronautics-space-administration.env: -------------------------------------------------------------------------------- 1 | # National Aeronautics and Space Administration 2 | export ANALYTICS_REPORT_IDS="ga:68619810" 3 | export AGENCY_NAME=national-aeronautics-space-administration 4 | export AWS_BUCKET_PATH=data/national-aeronautics-space-administration 5 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:7.8-alpine 2 | RUN mkdir -p /usr/src/app 3 | WORKDIR /usr/src/app 4 | 5 | ARG NODE_ENV 6 | ENV NODE_ENV $NODE_ENV 7 | 8 | COPY package.json /usr/src/app/ 9 | 10 | RUN npm install && \ 11 | npm cache clean 12 | 13 | COPY . /usr/src/app 14 | RUN npm link 15 | 16 | ENTRYPOINT ["analytics"] 17 | -------------------------------------------------------------------------------- /test/support/mocks/googleapis-auth.js: -------------------------------------------------------------------------------- 1 | const dataFixture = require("../fixtures/data") 2 | 3 | const googleAPIsMock = () => { 4 | function JWT() { 5 | this.initArguments = arguments 6 | } 7 | JWT.prototype.authorize = (callback) => callback(null, {}) 8 | return { auth: { JWT } } 9 | } 10 | 11 | module.exports = googleAPIsMock 12 | -------------------------------------------------------------------------------- /test/support/fixtures/report.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | "name": "today", 3 | "frequency": "hourly", 4 | "query": { 5 | "dimensions": ["ga:date", "ga:hour"], 6 | "metrics": ["ga:sessions"], 7 | "start-date": "today", 8 | "end-date": "today", 9 | }, 10 | "meta": { 11 | "name": "Today", 12 | "description": "Today's visits for all sites." 13 | }, 14 | } 15 | -------------------------------------------------------------------------------- /knexfile.js: -------------------------------------------------------------------------------- 1 | const config = require("./src/config") 2 | 3 | module.exports = { 4 | development: { 5 | client: 'postgresql', 6 | connection: config.postgres, 7 | }, 8 | test: { 9 | client: 'postgresql', 10 | connection: { 11 | database: process.env.CIRCLECI ? "circle_test" : "analytics_reporter_test", 12 | }, 13 | migrations: { 14 | tableName: 'knex_migrations', 15 | }, 16 | }, 17 | } 18 | -------------------------------------------------------------------------------- /bin/deploy-ci.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | # Install CF 6 | curl https://s3-us-west-1.amazonaws.com/cf-cli-releases/releases/v6.25.0/cf-cli_6.25.0_linux_x86-64.tgz | tar xzvf - 7 | mv cf /home/ubuntu/bin/cf 8 | 9 | # Log into cloud.gov 10 | cf api api.fr.cloud.gov 11 | cf login -u $CF_USERNAME -p $CF_PASSWORD -o gsa-opp-analytics -s analytics-dev 12 | 13 | # Push the app 14 | cf push analytics-reporter 15 | 16 | cf logout 17 | -------------------------------------------------------------------------------- /migrations/20170308164751_create_analytics_data.js: -------------------------------------------------------------------------------- 1 | exports.up = function(knex) { 2 | return knex.schema.createTable("analytics_data", table => { 3 | table.increments("id") 4 | table.string("report_name") 5 | table.string("report_agency") 6 | table.dateTime("date_time") 7 | table.jsonb("data") 8 | table.timestamps(true, true) 9 | }) 10 | }; 11 | 12 | exports.down = function(knex) { 13 | return knex.schema.dropTable('analytics_data') 14 | }; 15 | -------------------------------------------------------------------------------- /src/publish/disk.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs") 2 | const path = require("path") 3 | 4 | const publish = (report, results, { output, format }) => { 5 | const filename = `${report.name}.${format}` 6 | const filepath = path.join(output, filename) 7 | 8 | return new Promise((resolve, reject) => { 9 | fs.writeFile(filepath, results, err => { 10 | if (err) { 11 | reject(err) 12 | } else { 13 | resolve() 14 | } 15 | }) 16 | }) 17 | } 18 | 19 | module.exports = { publish } 20 | -------------------------------------------------------------------------------- /test/support/mocks/googleapis-analytics.js: -------------------------------------------------------------------------------- 1 | const dataFixture = require("../fixtures/data") 2 | 3 | const googleAPIsMock = () => { 4 | const data = Object.assign({}, dataFixture) 5 | const realtime = { get: (query, callback) => callback(null, data) } 6 | const ga = { get: (query, callback) => callback(null, data) } 7 | 8 | const analytics = (() => ({ 9 | data: { 10 | realtime: realtime, 11 | ga: ga, 12 | } 13 | })) 14 | 15 | return { realtime, ga, analytics } 16 | } 17 | 18 | module.exports = googleAPIsMock 19 | -------------------------------------------------------------------------------- /migrations/20170522094056_rename_date_time_to_date.js: -------------------------------------------------------------------------------- 1 | 2 | exports.up = function(knex, Promise) { 3 | return knex.schema.raw("ALTER TABLE analytics_data RENAME COLUMN date_time TO date").then(() => { 4 | return knex.schema.raw("ALTER TABLE analytics_data ALTER COLUMN date TYPE date") 5 | }) 6 | }; 7 | 8 | exports.down = function(knex, Promise) { 9 | return knex.schema.raw("ALTER TABLE analytics_data RENAME COLUMN date TO date_time").then(() => { 10 | return knex.schema.raw("ALTER TABLE analytics_data ALTER COLUMN date_time TYPE timestamp with time zone") 11 | }) 12 | }; 13 | -------------------------------------------------------------------------------- /test/support/database.js: -------------------------------------------------------------------------------- 1 | const { ANALYTICS_DATA_TABLE_NAME } = require("../../src/publish/postgres") 2 | 3 | const knex = require("knex") 4 | 5 | const connection = { 6 | host: process.env.PG_HOST ? process.env.PG_HOST : "localhost", 7 | database: process.env.CIRCLECI ? "circle_test" : "analytics_reporter_test", 8 | user : process.env.PG_USER ? process.env.PG_USER : 'postgres' 9 | } 10 | 11 | const resetSchema = () => { 12 | const db = knex({ client: "pg", connection }) 13 | return db("analytics_data").delete() 14 | } 15 | 16 | module.exports = { connection, resetSchema } 17 | -------------------------------------------------------------------------------- /migrations/20170316115145_add_analytics_data_indexes.js: -------------------------------------------------------------------------------- 1 | exports.up = function(knex) { 2 | return knex.schema.table("analytics_data", table => { 3 | table.index(["report_name", "report_agency"]) 4 | }).then(() => { 5 | return knex.schema.raw("CREATE INDEX analytics_data_date_time_desc ON analytics_data (date_time DESC NULLS LAST)") 6 | }) 7 | }; 8 | 9 | exports.down = function(knex, Promise) { 10 | return knex.schema.table("analytics_data", table => { 11 | table.dropIndex(["report_name", "report_agency"]) 12 | table.dropIndex("date_time", "analytics_data_date_time_desc") 13 | }) 14 | }; 15 | -------------------------------------------------------------------------------- /src/google-analytics/credential-loader.js: -------------------------------------------------------------------------------- 1 | const config = require("../config") 2 | 3 | global.analyticsCredentialsIndex = 0 4 | 5 | const loadCredentials = () => { 6 | const credentialData = JSON.parse(config.analytics_credentials) 7 | const credentialsArray = _wrapArray(credentialData) 8 | const index = global.analyticsCredentialsIndex++ % credentialsArray.length 9 | return credentialsArray[index] 10 | } 11 | 12 | const _wrapArray = (object) => { 13 | if (Array.isArray(object)) { 14 | return object 15 | } else { 16 | return [object] 17 | } 18 | } 19 | 20 | module.exports = { loadCredentials } 21 | -------------------------------------------------------------------------------- /bin/docker-update: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if docker-compose build; then 4 | if docker-compose run reporter ./bin/update; then 5 | echo 6 | echo "Docker environment updated successfully." 7 | exit 0 8 | fi 9 | fi 10 | 11 | echo 12 | echo "Alas, something didn't work when trying to update your Docker setup." 13 | echo "If you're not sure what the problem is, you might want to just " 14 | echo "reset your environment by running:" 15 | echo 16 | echo " docker-compose down -v" 17 | echo " $0" 18 | echo 19 | echo "Note that this will reset your database! It will also re-fetch" 20 | echo "your package dependencies, among other things, so make sure you" 21 | echo "have a good internet connection." 22 | 23 | exit 1 24 | -------------------------------------------------------------------------------- /bin/analytics: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | /* 4 | * Run all analytics reports output JSON to disk. 5 | * 6 | * Usage: analytics 7 | * 8 | * Defaults to printing JSON to STDOUT. 9 | * 10 | * --output: Output to a directory. 11 | * --publish: Publish to an S3 bucket. 12 | * --only: only run one or more named reports. 13 | * --slim: Where supported, use totals only (omit the `data` array). 14 | * Only applies to JSON, and reports where "slim": true. 15 | * --csv: CSV instead of JSON. 16 | * --frequency: Limit to reports with this 'frequency' value. 17 | * --debug: print debug details on STDOUT 18 | */ 19 | 20 | const minimist = require("minimist"); 21 | const run = require("../index.js").run; 22 | const options = minimist(process.argv.slice(2)); 23 | 24 | run(options); 25 | -------------------------------------------------------------------------------- /src/google-analytics/query-builder.js: -------------------------------------------------------------------------------- 1 | const config = require('../config') 2 | 3 | const buildQuery = (report) => { 4 | let query = Object.assign({}, report.query) 5 | query = buildQueryArrays(query) 6 | query.samplingLevel = "HIGHER_PRECISION"; 7 | query['max-results'] = query['max-results'] || 10000; 8 | query.ids = config.account.ids; 9 | return query 10 | } 11 | 12 | const buildQueryArrays = (query) => { 13 | query = Object.assign({}, query) 14 | if (query.dimensions) { 15 | query.dimensions = query.dimensions.join(",") 16 | } 17 | if (query.metrics) { 18 | query.metrics = query.metrics.join(",") 19 | } 20 | if (query.filters) { 21 | query.filters = query.filters.join(";") 22 | } 23 | return query 24 | } 25 | 26 | module.exports = { buildQuery } 27 | -------------------------------------------------------------------------------- /src/analytics.js: -------------------------------------------------------------------------------- 1 | const path = require("path") 2 | const config = require('./config') 3 | const GoogleAnalyticsClient = require("./google-analytics/client") 4 | const GoogleAnalyticsDataProcessor = require("./process-results/ga-data-processor") 5 | 6 | const query = (report) => { 7 | if (!report) { 8 | return Promise.reject(new Error("Analytics.query missing required argument `report`")) 9 | } 10 | 11 | return GoogleAnalyticsClient.fetchData(report).then(data => { 12 | return GoogleAnalyticsDataProcessor.processData(report, data) 13 | }) 14 | } 15 | 16 | const _loadReports = () => { 17 | const _reportFilePath = path.resolve(process.cwd(), config.reports_file || "reports/reports.json") 18 | return require(_reportFilePath).reports 19 | } 20 | 21 | module.exports = { 22 | query, 23 | reports: _loadReports(), 24 | } 25 | -------------------------------------------------------------------------------- /env.example: -------------------------------------------------------------------------------- 1 | export ANALYTICS_REPORT_EMAIL="YYYYYYY@developer.gserviceaccount.com" 2 | export ANALYTICS_REPORT_IDS="ga:XXXXXX" 3 | 4 | export ANALYTICS_CREDENTIALS='[ 5 | { 6 | "key": "{PRIVATE 1 KEY HERE}", 7 | "email": "email_1@example.com" 8 | }, 9 | { 10 | "key": "{PRIVATE 2 KEY HERE}", 11 | "email": "email_2@example.com" 12 | } 13 | ]' 14 | 15 | # Optional: if your profile uses a single hostname, set it here. 16 | # It will be prepended to page paths. 17 | # export ANALYTICS_HOSTNAME=https://example.gov 18 | 19 | # Optional: defaults to reports/reports.json inside module 20 | # export ANALYTICS_REPORTS_PATH=/path/to/report.json 21 | 22 | export AWS_REGION=us-east-1 23 | export AWS_ACCESS_KEY_ID=[your-key] 24 | export AWS_SECRET_ACCESS_KEY=[your-secret-key] 25 | 26 | export AWS_BUCKET=[your-bucket] 27 | export AWS_BUCKET_PATH=[your-path] 28 | export AWS_CACHE_TIME=0 29 | -------------------------------------------------------------------------------- /src/process-results/result-formatter.js: -------------------------------------------------------------------------------- 1 | const csv = require("fast-csv") 2 | 3 | const formatResult = (result, { format = "json", slim = false } = {}) => { 4 | result = Object.assign({}, result) 5 | 6 | switch(format) { 7 | case "json": 8 | return _formatJSON(result, { slim }) 9 | break 10 | case "csv": 11 | return _formatCSV(result) 12 | break 13 | default: 14 | return Promise.reject("Unsupported format: " + format) 15 | } 16 | } 17 | 18 | const _formatJSON = (result, { slim }) => { 19 | if (slim) { 20 | delete result.data 21 | } 22 | return Promise.resolve(JSON.stringify(result, null, 2)) 23 | } 24 | 25 | const _formatCSV = (result) => { 26 | return new Promise((resolve, reject) => { 27 | csv.writeToString(result.data, {headers: true}, (err, data) => { 28 | if (err) { 29 | reject(err) 30 | } else { 31 | resolve(data) 32 | } 33 | }) 34 | }) 35 | } 36 | 37 | module.exports = { formatResult } 38 | -------------------------------------------------------------------------------- /src/google-analytics/client.js: -------------------------------------------------------------------------------- 1 | const google = require("googleapis") 2 | const GoogleAnalyticsQueryAuthorizer = require("./query-authorizer") 3 | const GoogleAnalyticsQueryBuilder = require("./query-builder") 4 | 5 | const fetchData = (report) => { 6 | const query = GoogleAnalyticsQueryBuilder.buildQuery(report) 7 | return GoogleAnalyticsQueryAuthorizer.authorizeQuery(query).then(query => { 8 | return _executeFetchDataRequest(query, { realtime: report.realtime }) 9 | }) 10 | } 11 | 12 | const _executeFetchDataRequest = (query, { realtime }) => { 13 | return new Promise((resolve, reject) => { 14 | _get(realtime)(query, (err, data) => { 15 | if (err) { 16 | reject(err) 17 | } else { 18 | resolve(data) 19 | } 20 | }) 21 | }) 22 | } 23 | 24 | const _get = (realtime) => { 25 | const analytics = google.analytics("v3") 26 | if (realtime) { 27 | return analytics.data.realtime.get 28 | } else { 29 | return analytics.data.ga.get 30 | } 31 | } 32 | 33 | module.exports = { fetchData } 34 | -------------------------------------------------------------------------------- /manifest.yml: -------------------------------------------------------------------------------- 1 | applications: 2 | - name: analytics-reporter 3 | instances: 1 4 | memory: 512M 5 | disk_quota: 1024M 6 | no-route: true 7 | health-check-type: process 8 | buildpack: nodejs_buildpack 9 | command: node deploy/cron.js 10 | # These are examples. These variables are set via `cg-toolkit/generate-env.sh` 11 | # using the `.env` file referenced in README.md. 12 | # env: 13 | # ANALYTICS_KEY_PATH: 14 | # ANALYTICS_REPORT_EMAIL: 15 | # ANALYTICS_REPORT_IDS: 16 | # ANALYTICS_REPORTS_PATH: 17 | # AWS_ACCESS_KEY_ID: 18 | # AWS_BUCKET: 19 | # AWS_BUCKET_PATH: 20 | # AWS_CACHE_TIME: 21 | # AWS_DEFAULT_REGION: 22 | # AWS_REGION: 23 | # AWS_SECRET_ACCESS_KEY: 24 | # BUCKET_NAME: 25 | services: 26 | - analytics-s3 27 | - analytics-env 28 | - analytics-reporter-database 29 | stack: cflinuxfs2 30 | timeout: 180 31 | path: . 32 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.0" 2 | services: 3 | db: 4 | image: postgres:9.6 5 | environment: 6 | - POSTGRES_DB=analytics-reporter 7 | - POSTGRES_USER=analytics 8 | volumes: 9 | - pgdata:/var/lib/postgresql/data/ 10 | reporter: 11 | image: node:7.8 12 | entrypoint: "" 13 | command: "node ./deploy/cron.js" 14 | environment: 15 | - ANALYTICS_ROOT_PATH=/usr/src/app 16 | - ANALYTICS_CREDENTIALS=${ANALYTICS_CREDENTIALS} 17 | - ANALYTICS_REPORTS_PATH=${ANALYTICS_REPORTS_PATH} 18 | - ANALYTICS_REPORT_IDS=${ANALYTICS_REPORT_IDS} 19 | - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} 20 | - AWS_BUCKET=${AWS_BUCKET} 21 | - AWS_BUCKET_PATH=${AWS_BUCKET_PATH} 22 | - AWS_CACHE_TIME=${AWS_CACHE_TIME} 23 | - AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION} 24 | - AWS_REGION=${AWS_REGION} 25 | - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} 26 | - BUCKET_NAME=${BUCKET_NAME} 27 | - POSTGRES_HOST=db 28 | - POSTGRES_USER=analytics 29 | - POSTGRES_DATABASE=analytics-reporter 30 | links: 31 | - db 32 | working_dir: /usr/src/app 33 | volumes: 34 | - .:/usr/src/app 35 | - node_modules:/usr/src/app/node_modules 36 | volumes: 37 | node_modules: 38 | pgdata: 39 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Welcome! 2 | 3 | We're so glad you're thinking about contributing to an 18F open source project! If you're unsure or afraid of anything, just ask or submit the issue or pull request anyways. The worst that can happen is that you'll be politely asked to change something. We appreciate any sort of contribution, and don't want a wall of rules to get in the way of that. 4 | 5 | Before contributing, we encourage you to read our CONTRIBUTING policy (you are here), our LICENSE, and our README, all of which should be in this repository. If you have any questions, or want to read more about our underlying policies, you can consult the 18F Open Source Policy GitHub repository at https://github.com/18f/open-source-policy, or just shoot us an email/official government letterhead note to [18f@gsa.gov](mailto:18f@gsa.gov). 6 | 7 | ## Public domain 8 | 9 | This project is in the public domain within the United States, and 10 | copyright and related rights in the work worldwide are waived through 11 | the [CC0 1.0 Universal public domain dedication](https://creativecommons.org/publicdomain/zero/1.0/). 12 | 13 | All contributions to this project will be released under the CC0 14 | dedication. By submitting a pull request, you are agreeing to comply 15 | with this waiver of copyright interest. 16 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | As a work of the United States Government, this project is in the 2 | public domain within the United States. 3 | 4 | Additionally, we waive copyright and related rights in the work 5 | worldwide through the CC0 1.0 Universal public domain dedication. 6 | 7 | ## CC0 1.0 Universal Summary 8 | 9 | This is a human-readable summary of the [Legal Code (read the full text)](https://creativecommons.org/publicdomain/zero/1.0/legalcode). 10 | 11 | ### No Copyright 12 | 13 | The person who associated a work with this deed has dedicated the work to 14 | the public domain by waiving all of his or her rights to the work worldwide 15 | under copyright law, including all related and neighboring rights, to the 16 | extent allowed by law. 17 | 18 | You can copy, modify, distribute and perform the work, even for commercial 19 | purposes, all without asking permission. 20 | 21 | ### Other Information 22 | 23 | In no way are the patent or trademark rights of any person affected by CC0, 24 | nor are the rights that other persons may have in the work or in how the 25 | work is used, such as publicity or privacy rights. 26 | 27 | Unless expressly stated otherwise, the person who associated a work with 28 | this deed makes no warranties about the work, and disclaims liability for 29 | all uses of the work, to the fullest extent permitted by applicable law. 30 | When using or citing the work, you should not imply endorsement by the 31 | author or the affirmer. 32 | -------------------------------------------------------------------------------- /test/support/fixtures/data.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | kind: 'analytics#gaData', 3 | id: 'https://www.googleapis.com/analytics/v3/data/ga?ids=ga:96302018&dimensions=ga:date,ga:hour&metrics=ga:sessions&start-date=today&end-date=today&max-results=10000', 4 | query: { 5 | 'start-date': 'today', 'end-date': 'today', ids: 'ga:96302018', 6 | dimensions: 'ga:date,ga:hour', metrics: [ 'ga:sessions' ], 7 | 'start-index': 1, 'max-results': 10000, samplingLevel: 'HIGHER_PRECISION', 8 | }, 9 | itemsPerPage: 10000, 10 | totalResults: 24, 11 | selfLink: 'https://www.googleapis.com/analytics/v3/data/ga?ids=ga:96302018&dimensions=ga:date,ga:hour&metrics=ga:sessions&start-date=today&end-date=today&max-results=10000', 12 | profileInfo: { 13 | profileId: '96302018', 14 | accountId: '33523145', 15 | webPropertyId: 'UA-33523145-1', 16 | internalWebPropertyId: '60822123', 17 | profileName: 'Z3. Adjusted Gov-Wide Reporting Profile (.gov & .mil only)', 18 | tableId: 'ga:96302018' 19 | }, 20 | containsSampledData: false, 21 | columnHeaders: [ 22 | { name: 'ga:date', columnType: 'DIMENSION', dataType: 'STRING' }, 23 | { name: 'ga:hour', columnType: 'DIMENSION', dataType: 'STRING' }, 24 | { name: 'ga:sessions', columnType: 'METRIC', dataType: 'INTEGER' } 25 | ], 26 | totalsForAllResults: { 'ga:sessions': '6782212' }, 27 | rows: Array(24).fill(100).map((val, index) => { 28 | return ["20170130", `${index}`.length < 2 ? `0${index}` : `${index}`, `${val}`] 29 | }), 30 | } 31 | -------------------------------------------------------------------------------- /src/publish/s3.js: -------------------------------------------------------------------------------- 1 | const AWS = require("aws-sdk") 2 | const winston = require("winston-color") 3 | const zlib = require("zlib") 4 | const config = require("../config") 5 | 6 | // This is the case where using custom s3 api-like services like minio. 7 | const conf = { 8 | accessKeyId: config.aws.accessKeyId, 9 | secretAccessKey: config.aws.secretAccessKey, 10 | endpoint: config.aws.endpoint, 11 | s3ForcePathStyle: config.aws.s3ForcePathStyle, 12 | signatureVersion: config.aws.signatureVersion 13 | } 14 | 15 | const S3 = new AWS.S3(conf) 16 | const publish = (report, results, { format }) => { 17 | 18 | winston.debug("[" + report.name + "] Publishing to " + config.aws.bucket + "...") 19 | 20 | return _compress(results).then(compressed => { 21 | return S3.putObject({ 22 | Bucket: config.aws.bucket, 23 | Key: config.aws.path + "/" + report.name + "." + format, 24 | Body: compressed, 25 | ContentType: _mime(format), 26 | ContentEncoding: "gzip", 27 | ACL: "public-read", 28 | CacheControl: "max-age=" + (config.aws.cache || 0), 29 | }).promise() 30 | }) 31 | } 32 | 33 | const _compress = (data) => { 34 | return new Promise((resolve, reject) => { 35 | zlib.gzip(data, (err, compressed) => { 36 | if (err) { 37 | reject(err) 38 | } else { 39 | resolve(compressed) 40 | } 41 | }) 42 | }) 43 | } 44 | 45 | const _mime = (format) => { 46 | return { 47 | json: "application/json", 48 | csv: "text/csv", 49 | }[format] 50 | } 51 | 52 | module.exports = { publish } 53 | -------------------------------------------------------------------------------- /test/support/fixtures/data_with_hostname.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | kind: 'analytics#gaData', 3 | id: 'https://www.googleapis.com/analytics/v3/data/ga?ids=ga:96302018&dimensions=ga:date,ga:hour&metrics=ga:sessions&start-date=today&end-date=today&max-results=10000', 4 | query: { 5 | 'start-date': 'today', 'end-date': 'today', ids: 'ga:96302018', 6 | dimensions: 'ga:date,ga:hour', metrics: [ 'ga:sessions' ], 7 | 'start-index': 1, 'max-results': 10000, samplingLevel: 'HIGHER_PRECISION', 8 | }, 9 | itemsPerPage: 10000, 10 | totalResults: 24, 11 | selfLink: 'https://www.googleapis.com/analytics/v3/data/ga?ids=ga:96302018&dimensions=ga:date,ga:hour&metrics=ga:sessions&start-date=today&end-date=today&max-results=10000', 12 | profileInfo: { 13 | profileId: '96302018', 14 | accountId: '33523145', 15 | webPropertyId: 'UA-33523145-1', 16 | internalWebPropertyId: '60822123', 17 | profileName: 'Z3. Adjusted Gov-Wide Reporting Profile (.gov & .mil only)', 18 | tableId: 'ga:96302018' 19 | }, 20 | containsSampledData: false, 21 | columnHeaders: [ 22 | { name: 'ga:date', columnType: 'DIMENSION', dataType: 'STRING' }, 23 | { name: 'ga:hour', columnType: 'DIMENSION', dataType: 'STRING' }, 24 | { name: 'ga:hostname', columnType: 'DIMENSION', dataType: 'STRING' }, 25 | { name: 'ga:sessions', columnType: 'METRIC', dataType: 'INTEGER' } 26 | ], 27 | totalsForAllResults: { 'ga:sessions': '6782212' }, 28 | rows: Array(24).fill(100).map((val, index) => { 29 | return ["20170130", `${index}`.length < 2 ? `0${index}` : `${index}`, `www.example${index}.com`,`${val}`] 30 | }), 31 | } 32 | -------------------------------------------------------------------------------- /src/google-analytics/query-authorizer.js: -------------------------------------------------------------------------------- 1 | const googleapis = require('googleapis') 2 | const fs = require('fs') 3 | const config = require('../config') 4 | const GoogleAnalyticsCredentialLoader = require("./credential-loader") 5 | 6 | const authorizeQuery = (query) => { 7 | const credentials = _getCredentials() 8 | const email = credentials.email 9 | const key = credentials.key 10 | const scopes = ['https://www.googleapis.com/auth/analytics.readonly'] 11 | const jwt = new googleapis.auth.JWT(email, null, key, scopes); 12 | 13 | query = Object.assign({}, query, { auth: jwt }) 14 | 15 | return new Promise((resolve, reject) => { 16 | jwt.authorize((err, result) => { 17 | if (err) { 18 | reject(err) 19 | } else { 20 | resolve(query) 21 | } 22 | }) 23 | }) 24 | } 25 | 26 | const _getCredentials = () => { 27 | if (config.key) { 28 | return { key: config.key, email: config.email } 29 | } else if (config.key_file) { 30 | return _loadCredentialsFromKeyfile(config.key_file) 31 | } else if (config.analytics_credentials) { 32 | return GoogleAnalyticsCredentialLoader.loadCredentials() 33 | } else { 34 | throw new Error("No key or key file specified in config") 35 | } 36 | } 37 | 38 | const _loadCredentialsFromKeyfile = (keyfile) => { 39 | if (!fs.existsSync(keyfile)) { 40 | throw new Error(`No such key file: ${keyfile}`) 41 | } 42 | 43 | let key = fs.readFileSync(keyfile).toString().trim() 44 | let email = config.email 45 | 46 | if (keyfile.match(/\.json$/)) { 47 | const json = JSON.parse(key) 48 | key = json.private_key 49 | email = json.client_email 50 | } 51 | return { key, email } 52 | } 53 | 54 | module.exports = { authorizeQuery } 55 | -------------------------------------------------------------------------------- /src/config.js: -------------------------------------------------------------------------------- 1 | // Set environment variables to configure the application. 2 | 3 | module.exports = { 4 | 5 | email: process.env.ANALYTICS_REPORT_EMAIL, 6 | key: process.env.ANALYTICS_KEY, 7 | key_file: process.env.ANALYTICS_KEY_PATH, 8 | analytics_credentials: process.env.ANALYTICS_CREDENTIALS, 9 | 10 | reports_file: process.env.ANALYTICS_REPORTS_PATH, 11 | 12 | debug: (process.env.ANALYTICS_DEBUG ? true : false), 13 | 14 | /* 15 | AWS S3 information. 16 | 17 | Separately, you need to set AWS_REGION, AWS_ACCESS_KEY_ID, and 18 | AWS_SECRET_ACCESS_KEY. The AWS SDK for Node reads these in automatically. 19 | */ 20 | aws: { 21 | // No trailing slashes 22 | bucket: process.env.AWS_BUCKET, 23 | path: process.env.AWS_BUCKET_PATH, 24 | // HTTP cache time in seconds. Defaults to 0. 25 | cache: process.env.AWS_CACHE_TIME, 26 | endpoint: process.env.AWS_S3_ENDPOINT, 27 | accessKeyId: process.env.AWS_ACCESS_KEY_ID, 28 | secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, 29 | s3ForcePathStyle: process.env.AWS_S3_FORCE_STYLE_PATH, 30 | signatureVersion: process.env.AWS_SIGNATURE_VERSION 31 | }, 32 | 33 | account: { 34 | ids: process.env.ANALYTICS_REPORT_IDS, 35 | agency_name: process.env.AGENCY_NAME, 36 | // needed for realtime reports which don't include hostname 37 | // leave blank if your view includes hostnames 38 | hostname: process.env.ANALYTICS_HOSTNAME || "", 39 | }, 40 | 41 | postgres: { 42 | host : process.env.POSTGRES_HOST, 43 | user : process.env.POSTGRES_USER, 44 | password : process.env.POSTGRES_PASSWORD, 45 | database : process.env.POSTGRES_DATABASE || "analytics-reporter", 46 | }, 47 | 48 | static: { 49 | path: '../analytics.usa.gov/', 50 | }, 51 | }; 52 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "analytics-reporter", 3 | "version": "1.2.2", 4 | "description": "A lightweight command line tool for reporting and publishing analytics data from a Google Analytics account.", 5 | "keywords": [ 6 | "analytics", 7 | "google analytics" 8 | ], 9 | "homepage": "https://github.com/18F/analytics-reporter", 10 | "license": "CC0-1.0", 11 | "scripts": { 12 | "migrate": "`npm bin`/knex migrate:latest", 13 | "pretest": "NODE_ENV=test npm run migrate", 14 | "start": "node app.js", 15 | "test": "`npm bin`/mocha test/*.test.js test/**/*.test.js" 16 | }, 17 | "contributors": [ 18 | { 19 | "name": "Gabriel Ramirez", 20 | "email": "gabriel.ramirez@gsa.gov" 21 | }, 22 | { 23 | "name": "Eric Mill", 24 | "email": "eric.mill@gsa.gov" 25 | }, 26 | { 27 | "name": "Lauren Ancona", 28 | "email": "lauren.ancona@phila.gov" 29 | }, 30 | { 31 | "name": "Eric Schles", 32 | "email": "eric.schles@gsa.gov" 33 | } 34 | ], 35 | "files": [ 36 | "bin", 37 | "src", 38 | "test", 39 | "reports", 40 | "index.js", 41 | "newrelic.js", 42 | "package.json", 43 | "*.md" 44 | ], 45 | "engines": { 46 | "node": ">=7.0.0" 47 | }, 48 | "preferGlobal": true, 49 | "main": "index", 50 | "bin": { 51 | "analytics": "./bin/analytics" 52 | }, 53 | "repository": { 54 | "type": "git", 55 | "url": "git://github.com/18F/analytics-reporter.git" 56 | }, 57 | "bugs": { 58 | "url": "https://github.com/18F/analytics-reporter/issues" 59 | }, 60 | "dependencies": { 61 | "aws-sdk": "2.58.0", 62 | "fast-csv": "^2.4.0", 63 | "googleapis": "^19.0.0", 64 | "minimist": "^1.2.0", 65 | "winston-color": "^1.0.0" 66 | }, 67 | "devDependencies": { 68 | "chai": "^4.0.2", 69 | "mocha": "^3.2.0", 70 | "proxyquire": "^1.7.11" 71 | }, 72 | "optionalDependencies": { 73 | "knex": "^0.13.0", 74 | "newrelic": "^1.36.1", 75 | "pg": "^6.1.2" 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /test/publish/disk.test.js: -------------------------------------------------------------------------------- 1 | const expect = require("chai").expect 2 | const proxyquire = require("proxyquire") 3 | 4 | describe("DiskPublisher", () => { 5 | let DiskPublisher 6 | let fs = {} 7 | 8 | beforeEach(() => { 9 | fs = { writeFile: (path, contents, cb) => cb() } 10 | DiskPublisher = proxyquire("../../src/publish/disk", { 11 | fs: fs, 12 | }) 13 | }) 14 | 15 | describe(".publish(report, results, options)", () => { 16 | context("when the format is json", () => { 17 | it("should write the results to /.json", done => { 18 | const options = { output: "path/to/output", format: "json" } 19 | const report = { name: "report-name" } 20 | const results = "I'm the results" 21 | 22 | let fileWritten = false 23 | fs.writeFile = (path, contents, cb) => { 24 | expect(path).to.equal("path/to/output/report-name.json") 25 | expect(contents).to.equal("I'm the results") 26 | fileWritten = true 27 | cb(null) 28 | } 29 | 30 | DiskPublisher.publish(report, results, options).then(() => { 31 | expect(fileWritten).to.be.true 32 | done() 33 | }).catch(done) 34 | }) 35 | }) 36 | 37 | context("when the format is csv", () => { 38 | it("should write the results to /.csv", done => { 39 | const options = { output: "path/to/output", format: "csv" } 40 | const report = { name: "report-name" } 41 | const results = "I'm the results" 42 | 43 | let fileWritten = false 44 | fs.writeFile = (path, contents, cb) => { 45 | expect(path).to.equal("path/to/output/report-name.csv") 46 | expect(contents).to.equal("I'm the results") 47 | fileWritten = true 48 | cb(null) 49 | } 50 | 51 | DiskPublisher.publish(report, results, options).then(() => { 52 | expect(fileWritten).to.be.true 53 | done() 54 | }).catch(done) 55 | }) 56 | }) 57 | }) 58 | }) 59 | -------------------------------------------------------------------------------- /test/google-analytics/credential-loader.test.js: -------------------------------------------------------------------------------- 1 | const expect = require("chai").expect 2 | const proxyquire = require("proxyquire") 3 | 4 | proxyquire.noCallThru() 5 | 6 | const config = {} 7 | 8 | const GoogleAnalyticsCredentialLoader = proxyquire("../../src/google-analytics/credential-loader", { 9 | "../config": config, 10 | }) 11 | 12 | describe("GoogleAnalyticsCredentialLoader", () => { 13 | describe(".loadCredentials()", () => { 14 | beforeEach(() => { 15 | config.analytics_credentials = undefined 16 | global.analyticsCredentialsIndex = 0 17 | }) 18 | 19 | it("should return the credentials if the credentials are an object", () => { 20 | config.analytics_credentials = `{ 21 | "email": "email@example.com", 22 | "key": "this-is-a-secret" 23 | }` 24 | 25 | const creds = GoogleAnalyticsCredentialLoader.loadCredentials() 26 | expect(creds.email).to.equal("email@example.com") 27 | expect(creds.key).to.equal("this-is-a-secret") 28 | }) 29 | 30 | it("should return successive credentials if the credentials are an array", () => { 31 | config.analytics_credentials = `[ 32 | { 33 | "email": "email_1@example.com", 34 | "key": "this-is-a-secret-1" 35 | }, 36 | { 37 | "email": "email_2@example.com", 38 | "key": "this-is-a-secret-2" 39 | } 40 | ]` 41 | 42 | const firstCreds = GoogleAnalyticsCredentialLoader.loadCredentials() 43 | const secondCreds = GoogleAnalyticsCredentialLoader.loadCredentials() 44 | const thirdCreds = GoogleAnalyticsCredentialLoader.loadCredentials() 45 | 46 | expect(firstCreds.email).to.equal("email_1@example.com") 47 | expect(firstCreds.key).to.equal("this-is-a-secret-1") 48 | expect(secondCreds.email).to.equal("email_2@example.com") 49 | expect(secondCreds.key).to.equal("this-is-a-secret-2") 50 | expect(thirdCreds.email).to.equal("email_1@example.com") 51 | expect(thirdCreds.key).to.equal("this-is-a-secret-1") 52 | }) 53 | }) 54 | }) 55 | -------------------------------------------------------------------------------- /test/analytics.test.js: -------------------------------------------------------------------------------- 1 | const expect = require("chai").expect 2 | const proxyquire = require("proxyquire") 3 | 4 | describe("Analytics", () => { 5 | let Analytics 6 | let GoogleAnalyticsClient 7 | let GoogleAnalyticsDataProcessor 8 | 9 | beforeEach(() => { 10 | GoogleAnalyticsClient = {} 11 | GoogleAnalyticsDataProcessor = {} 12 | 13 | Analytics = proxyquire("../src/analytics", { 14 | "./google-analytics/client": GoogleAnalyticsClient, 15 | "./process-results/ga-data-processor": GoogleAnalyticsDataProcessor, 16 | }) 17 | }) 18 | describe(".query(report)", () => { 19 | it("should resolve with formatted google analytics data for the given reports", done => { 20 | const report = { name: "Report Name" } 21 | const data = { rows: [1, 2, 3] } 22 | const processedData = { data: [1, 2, 3] } 23 | 24 | let fetchDataCalled = false 25 | let processedDataCalled = false 26 | 27 | GoogleAnalyticsClient.fetchData = (reportInput) => { 28 | fetchDataCalled = true 29 | expect(reportInput).to.equal(report) 30 | return Promise.resolve(data) 31 | } 32 | GoogleAnalyticsDataProcessor.processData = (reportInput, dataInput) => { 33 | processedDataCalled = true 34 | expect(reportInput).to.equal(report) 35 | expect(dataInput).to.equal(data) 36 | return Promise.resolve(processedData) 37 | } 38 | 39 | Analytics.query(report).then(results => { 40 | expect(results).to.equal(processedData) 41 | expect(fetchDataCalled).to.be.true 42 | expect(processedDataCalled).to.be.true 43 | done() 44 | }).catch(done) 45 | }) 46 | 47 | it("should reject if no report is provided", done => { 48 | Analytics.query().catch(err => { 49 | expect(err.message).to.equal("Analytics.query missing required argument `report`") 50 | done() 51 | }).catch(done) 52 | }) 53 | }) 54 | 55 | describe(".reports", () => { 56 | it("should load reports", () => { 57 | expect(Analytics.reports).to.be.an("array") 58 | }) 59 | }) 60 | }) 61 | -------------------------------------------------------------------------------- /test/process-results/result-formatter.test.js: -------------------------------------------------------------------------------- 1 | const csv = require("fast-csv") 2 | const expect = require("chai").expect 3 | const proxyquire = require("proxyquire") 4 | const reportFixture = require("../support/fixtures/report") 5 | const dataFixture = require("../support/fixtures/data") 6 | 7 | const GoogleAnalyticsDataProcessor = proxyquire("../../src/process-results/ga-data-processor", { 8 | "../config": { account: { hostname: "" } }, 9 | }) 10 | const ResultFormatter = require("../../src/process-results/result-formatter") 11 | 12 | describe("ResultFormatter", () => { 13 | describe("formatResult(result, options)", () => { 14 | let report 15 | let data 16 | 17 | beforeEach(() => { 18 | report = Object.assign({}, reportFixture) 19 | data = Object.assign({}, dataFixture) 20 | }) 21 | 22 | it("should format results into JSON if the format is 'json'", done => { 23 | const result = GoogleAnalyticsDataProcessor.processData(report, data) 24 | 25 | ResultFormatter.formatResult(result, { format: "json" }).then(formattedResult => { 26 | const object = JSON.parse(formattedResult) 27 | expect(object).to.deep.equal(object) 28 | done() 29 | }).catch(done) 30 | }) 31 | 32 | it("should remove the data attribute for JSON if options.slim is true", done => { 33 | const result = GoogleAnalyticsDataProcessor.processData(report, data) 34 | 35 | ResultFormatter.formatResult(result, { format: "json", slim: true }).then(formattedResult => { 36 | const object = JSON.parse(formattedResult) 37 | expect(object.data).to.be.undefined 38 | done() 39 | }).catch(done) 40 | }) 41 | 42 | it("should format results into CSV if the format is 'csv'", done => { 43 | const result = GoogleAnalyticsDataProcessor.processData(report, data) 44 | 45 | ResultFormatter.formatResult(result, { format: "csv", slim: true }).then(formattedResult => { 46 | csv.fromString(formattedResult, { strictColumnHandling: true }, { headers: true }) 47 | .on("invalid-data", (data) => { 48 | done(new Error("Invalid CSV data: " + data)) 49 | }) 50 | .on("finish", () => { 51 | done() 52 | }) 53 | }).catch(done) 54 | }) 55 | }) 56 | }) 57 | -------------------------------------------------------------------------------- /test/google-analytics/query-builder.test.js: -------------------------------------------------------------------------------- 1 | const expect = require("chai").expect 2 | const proxyquire = require("proxyquire") 3 | const reportFixture = require("../support/fixtures/report") 4 | 5 | proxyquire.noCallThru() 6 | 7 | const config = {} 8 | 9 | const GoogleAnalyticsQueryBuilder = proxyquire("../../src/google-analytics/query-builder", { 10 | "../config": config, 11 | }) 12 | 13 | describe("GoogleAnalyticsQueryBuilder", () => { 14 | describe(".buildQuery(report)", () => { 15 | let report 16 | 17 | beforeEach(() => { 18 | report = Object.assign({}, reportFixture) 19 | config.account = { 20 | ids: "ga:123456", 21 | } 22 | }) 23 | 24 | it("should set the properties from the query object on the report", () => { 25 | report.query = { 26 | a: "123abc", 27 | b: "456def", 28 | } 29 | 30 | const query = GoogleAnalyticsQueryBuilder.buildQuery(report) 31 | expect(query.a).to.equal("123abc") 32 | expect(query.b).to.equal("456def") 33 | }) 34 | 35 | it("should convert dimensions and metrics arrays into comma separated strings", () => { 36 | report.query.dimensions = ["ga:date", "ga:hour"] 37 | report.query.metrics = ["ga:sessions"] 38 | 39 | const query = GoogleAnalyticsQueryBuilder.buildQuery(report) 40 | expect(query.dimensions).to.equal("ga:date,ga:hour") 41 | expect(query.metrics).to.equal("ga:sessions") 42 | }) 43 | 44 | it("should convert filters array into a semicolon separated string", () => { 45 | report.query.filters = [ 46 | "ga:browser==Internet Explorer", 47 | "ga:operatingSystem==Windows", 48 | ] 49 | 50 | const query = GoogleAnalyticsQueryBuilder.buildQuery(report) 51 | expect(query.filters).to.equal( 52 | "ga:browser==Internet Explorer;ga:operatingSystem==Windows" 53 | ) 54 | }) 55 | 56 | it("should set the samplingLevel to HIGHER_PRECISION", () => { 57 | report.query.samplingLevel = undefined 58 | 59 | const query = GoogleAnalyticsQueryBuilder.buildQuery(report) 60 | expect(query.samplingLevel).to.equal("HIGHER_PRECISION") 61 | }) 62 | 63 | it("should set max-results if it is set on the report", () => { 64 | report.query["max-results"] = 3 65 | 66 | const query = GoogleAnalyticsQueryBuilder.buildQuery(report) 67 | expect(query["max-results"]).to.equal(3) 68 | }) 69 | 70 | it("should set max-results to 10000 if it is unset on the report", () => { 71 | report.query["max-results"] = undefined 72 | 73 | const query = GoogleAnalyticsQueryBuilder.buildQuery(report) 74 | expect(query["max-results"]).to.equal(10000) 75 | }) 76 | 77 | it("should set the ids to the account ids specified by the config", () => { 78 | config.account.ids = "ga:abc123" 79 | 80 | const query = GoogleAnalyticsQueryBuilder.buildQuery(report) 81 | expect(query.ids).to.equal("ga:abc123") 82 | }) 83 | }) 84 | }) 85 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs") 2 | const path = require('path') 3 | const Promise = require("bluebird") 4 | const winston = require("winston-color") 5 | 6 | const config = require("./src/config") 7 | const Analytics = require("./src/analytics") 8 | const DiskPublisher = require("./src/publish/disk") 9 | const PostgresPublisher = require("./src/publish/postgres") 10 | const ResultFormatter = require("./src/process-results/result-formatter") 11 | const S3Publisher = require("./src/publish/s3") 12 | 13 | winston.transports.console.level = "info" 14 | winston.transports.console.prettyPrint = true 15 | winston.transports.console.label = config.account.agency_name || "live" 16 | 17 | const run = function(options = {}) { 18 | if (options.debug || options.verbose) { 19 | winston.transports.console.level = "debug" 20 | } 21 | const reports = _filterReports(options) 22 | return Promise.each(reports, report => _runReport(report, options)) 23 | } 24 | 25 | const _filterReports = ({ only, frequency }) => { 26 | const reports = Analytics.reports 27 | if (only) { 28 | return reports.filter(report => report.name === only) 29 | } else if (frequency) { 30 | return reports.filter(report => report.frequency === frequency) 31 | } else { 32 | return reports 33 | } 34 | } 35 | 36 | const _optionsForReport = (report, options) => ({ 37 | format: options.csv ? "csv" : "json", 38 | output: options.output, 39 | publish: options.publish, 40 | realtime: report.realtime, 41 | slim: options.slim && report.slim, 42 | writeToDatabase: options["write-to-database"], 43 | }) 44 | 45 | const _publishReport = (report, formattedResult, options) => { 46 | winston.debug(`[${report.name}]`, "Publishing...") 47 | if (options.publish) { 48 | return S3Publisher.publish(report, formattedResult, options) 49 | } else if (options.output && typeof(options.output) === "string") { 50 | return DiskPublisher.publish(report, formattedResult, options) 51 | } else { 52 | console.log(formattedResult) 53 | } 54 | } 55 | 56 | const _runReport = (report, options) => { 57 | const reportOptions = _optionsForReport(report, options) 58 | winston.debug("[" + report.name + "] Fetching..."); 59 | 60 | return Analytics.query(report).then(results => { 61 | winston.debug("[" + report.name + "] Saving report data...") 62 | if (config.account.agency_name) { 63 | results.agency = config.account.agency_name 64 | } 65 | return _writeReportToDatabase(report, results, options) 66 | }).then(results => { 67 | return ResultFormatter.formatResult(results, reportOptions) 68 | }).then(formattedResult => { 69 | return _publishReport(report, formattedResult, reportOptions) 70 | }).catch(err => { 71 | winston.error(`[${report.name}] `, err) 72 | }) 73 | } 74 | 75 | const _writeReportToDatabase = (report, result, options) => { 76 | if (options["write-to-database"] && !report.realtime) { 77 | return PostgresPublisher.publish(result).then(() => result) 78 | } else { 79 | return Promise.resolve(result) 80 | } 81 | } 82 | 83 | module.exports = { run }; 84 | -------------------------------------------------------------------------------- /deploy/cron.js: -------------------------------------------------------------------------------- 1 | const spawn = require("child_process").spawn; 2 | const winston = require("winston-color") 3 | 4 | if (process.env.NEW_RELIC_APP_NAME) { 5 | winston.info("Starting New Relic") 6 | require("newrelic") 7 | } else { 8 | winston.warn("Skipping New Relic Activation") 9 | } 10 | 11 | const scriptRootPath = `${process.env.ANALYTICS_ROOT_PATH}/deploy` 12 | 13 | var api_run = function() { 14 | winston.info("about to run api.sh"); 15 | 16 | var api = spawn(`${scriptRootPath}/api.sh`) 17 | api.stdout.on("data", (data) => { 18 | winston.info("[api.sh]", data.toString().trim()) 19 | }) 20 | api.stderr.on("data", (data) => { 21 | winston.info("[api.sh]", data.toString().trim()) 22 | }) 23 | api.on("exit", (code) => { 24 | winston.info("api.sh exitted with code:", code) 25 | }) 26 | } 27 | 28 | var daily_run = function() { 29 | winston.info("about to run daily.sh"); 30 | 31 | var daily = spawn(`${scriptRootPath}/daily.sh`) 32 | daily.stdout.on("data", (data) => { 33 | winston.info("[daily.sh]", data.toString().trim()) 34 | }) 35 | daily.stderr.on("data", (data) => { 36 | winston.info("[daily.sh]", data.toString().trim()) 37 | }) 38 | daily.on("exit", (code) => { 39 | winston.info("daily.sh exitted with code:", code) 40 | }) 41 | } 42 | 43 | var hourly_run = function(){ 44 | winston.info("about to run hourly.sh"); 45 | 46 | var hourly = spawn(`${scriptRootPath}/hourly.sh`) 47 | hourly.stdout.on("data", (data) => { 48 | winston.info("[hourly.sh]", data.toString().trim()) 49 | }) 50 | hourly.stderr.on("data", (data) => { 51 | winston.info("[hourly.sh]", data.toString().trim()) 52 | }) 53 | hourly.on("exit", (code) => { 54 | winston.info("hourly.sh exitted with code:", code) 55 | }) 56 | } 57 | 58 | var realtime_run = function(){ 59 | winston.info("about to run realtime.sh"); 60 | 61 | var realtime = spawn(`${scriptRootPath}/realtime.sh`) 62 | realtime.stdout.on("data", (data) => { 63 | winston.info("[realtime.sh]", data.toString().trim()) 64 | }) 65 | realtime.stderr.on("data", (data) => { 66 | winston.info("[realtime.sh]", data.toString().trim()) 67 | }) 68 | realtime.on("exit", (code) => { 69 | winston.info("realtime.sh exitted with code:", code) 70 | }) 71 | } 72 | 73 | /** 74 | Daily reports run every morning at 10 AM UTC. 75 | This calculates the offset between now and then for the next scheduled run. 76 | */ 77 | var calculateNextDailyRunTimeOffset = function(){ 78 | const currentTime = new Date(); 79 | const nextRunTime = new Date( 80 | currentTime.getFullYear(), 81 | currentTime.getMonth(), 82 | currentTime.getDate() + 1, 83 | 10 - currentTime.getTimezoneOffset() / 60 84 | ); 85 | return (nextRunTime - currentTime) % (1000 * 60 * 60 * 24) 86 | } 87 | 88 | winston.info("starting cron.js!"); 89 | api_run(); 90 | daily_run(); 91 | hourly_run(); 92 | realtime_run(); 93 | //api 94 | setInterval(api_run,1000 * 60 * 60 * 24) 95 | //daily 96 | setTimeout(() => { 97 | // Run at 10 AM UTC, then every 24 hours afterwards 98 | daily_run(); 99 | setInterval(daily_run, 1000 * 60 * 60 * 24); 100 | }, calculateNextDailyRunTimeOffset()); 101 | //hourly 102 | setInterval(hourly_run,1000 * 60 * 60); 103 | //realtime 104 | setInterval(realtime_run,1000 * 60 * 5); 105 | -------------------------------------------------------------------------------- /src/publish/postgres.js: -------------------------------------------------------------------------------- 1 | const ANALYTICS_DATA_TABLE_NAME = "analytics_data" 2 | 3 | const knex = require("knex") 4 | const Promise = require("bluebird") 5 | const config = require("../config") 6 | 7 | const publish = (results) => { 8 | if (results.query.dimensions.match(/ga:date/)) { 9 | const db = knex({ client: "pg", connection: config.postgres }) 10 | return _writeRegularResults({ db, results }).then(() => db.destroy()) 11 | } else { 12 | return Promise.resolve() 13 | } 14 | } 15 | 16 | const _convertDataAttributesToNumbers = (data) => { 17 | const transformedData = Object.assign({}, data) 18 | 19 | const numbericalAttributes = ["visits", "total_events", "users"] 20 | numbericalAttributes.forEach(attributeName => { 21 | if (transformedData[attributeName]) { 22 | transformedData[attributeName] = Number(transformedData[attributeName]) 23 | } 24 | }) 25 | 26 | return transformedData 27 | } 28 | 29 | const _dataForDataPoint = (dataPoint) => { 30 | const data = _convertDataAttributesToNumbers(dataPoint) 31 | 32 | const date = _dateTimeForDataPoint(dataPoint) 33 | 34 | delete data.date 35 | delete data.hour 36 | 37 | return { 38 | date, 39 | data, 40 | } 41 | } 42 | 43 | const _dateTimeForDataPoint = (dataPoint) => { 44 | if (!isNaN(Date.parse(dataPoint.date))) { 45 | return dataPoint.date 46 | } 47 | } 48 | 49 | const _queryForExistingRow = ({ db, row }) => { 50 | query = db(ANALYTICS_DATA_TABLE_NAME) 51 | 52 | Object.keys(row).forEach(key => { 53 | if (row[key] === undefined) { 54 | return 55 | } else if (key === "data") { 56 | const dataQuery = Object.assign({}, row.data) 57 | delete dataQuery.visits 58 | delete dataQuery.users 59 | delete dataQuery.total_events 60 | Object.keys(dataQuery).forEach(dataKey => { 61 | query = query.whereRaw(`data->>'${dataKey}' = ?`, [dataQuery[dataKey]]) 62 | }) 63 | } else { 64 | query = query.where({ [key]: row[key] }) 65 | } 66 | }) 67 | 68 | return query.select() 69 | } 70 | 71 | const _handleExistingRow = ({ db, existingRow, newRow }) => { 72 | if (existingRow.data.visits != newRow.data.visits || 73 | existingRow.data.users != newRow.data.users || 74 | existingRow.data.total_events != newRow.data.total_events 75 | ) { 76 | return db(ANALYTICS_DATA_TABLE_NAME).where({ id: existingRow.id }).update(newRow) 77 | } 78 | } 79 | 80 | const _rowForDataPoint = ({ results, dataPoint }) => { 81 | const row = _dataForDataPoint(dataPoint) 82 | row.report_name = results.name 83 | row.report_agency = results.agency 84 | return row 85 | } 86 | 87 | const _writeRegularResults = ({ db, results }) => { 88 | const rows = results.data.map(dataPoint => { 89 | return _rowForDataPoint({ results, dataPoint }) 90 | }) 91 | 92 | const rowsToInsert = [] 93 | return Promise.each(rows, row => { 94 | return _queryForExistingRow({ db, row }).then(results => { 95 | if (row.date === undefined) { 96 | return 97 | } else if (results.length === 0) { 98 | rowsToInsert.push(row) 99 | } else if (results.length === 1) { 100 | return _handleExistingRow({ db, existingRow: results[0], newRow: row }) 101 | } 102 | }) 103 | }).then(() => { 104 | return db(ANALYTICS_DATA_TABLE_NAME).insert(rowsToInsert) 105 | }).then(() => { 106 | return db.destroy() 107 | }) 108 | } 109 | 110 | module.exports = { publish, ANALYTICS_DATA_TABLE_NAME } 111 | -------------------------------------------------------------------------------- /test/google-analytics/client.test.js: -------------------------------------------------------------------------------- 1 | const expect = require("chai").expect 2 | const proxyquire = require("proxyquire") 3 | const googleAPIsMock = require("../support/mocks/googleapis-analytics") 4 | 5 | proxyquire.noCallThru() 6 | 7 | let googleapis 8 | let GoogleAnalyticsQueryAuthorizer 9 | let GoogleAnalyticsQueryBuilder 10 | let report 11 | 12 | let GoogleAnalyticsClient 13 | 14 | describe("GoogleAnalyticsClient", () => { 15 | describe(".fetchData(query, options)", () => { 16 | beforeEach(() => { 17 | googleapis = googleAPIsMock() 18 | GoogleAnalyticsQueryAuthorizer = { authorizeQuery: (query) => Promise.resolve(query) } 19 | GoogleAnalyticsQueryBuilder = { buildQuery: () => ({}) } 20 | 21 | GoogleAnalyticsClient = proxyquire("../../src/google-analytics/client", { 22 | googleapis, 23 | "./query-authorizer": GoogleAnalyticsQueryAuthorizer, 24 | "./query-builder": GoogleAnalyticsQueryBuilder, 25 | }) 26 | }) 27 | 28 | it("should build and authorize a query and use that to call the google api", done => { 29 | const report = { name: "realtime" } 30 | 31 | let queryBuilderCalled = false 32 | GoogleAnalyticsQueryBuilder.buildQuery = (report) => { 33 | expect(report).to.deep.equal({ name: "realtime" }) 34 | queryBuilderCalled = true 35 | return { query: true } 36 | } 37 | 38 | let queryAuthorizerCalled = false 39 | GoogleAnalyticsQueryAuthorizer.authorizeQuery = (query) => { 40 | queryAuthorizerCalled = true 41 | expect(query).to.deep.equal({ query: true }) 42 | query.authorized = true 43 | return Promise.resolve(query) 44 | } 45 | 46 | let googleAPICalled = false 47 | googleapis.ga.get = (params, cb) => { 48 | googleAPICalled = true 49 | expect(params).to.deep.equal({ query: true, authorized: true }) 50 | cb(null, {}) 51 | } 52 | 53 | GoogleAnalyticsClient.fetchData(report).then(() => { 54 | expect(queryBuilderCalled).to.be.true 55 | expect(queryAuthorizerCalled).to.be.true 56 | expect(googleAPICalled).to.be.true 57 | done() 58 | }).catch(done) 59 | }) 60 | 61 | it("should return a promise for Google Analytics data", done => { 62 | googleapis.ga.get = (params, cb) => { 63 | cb(null, { data: "that's me" }) 64 | } 65 | 66 | GoogleAnalyticsClient.fetchData({}).then(result => { 67 | expect(result).to.deep.equal({ data: "that's me" }) 68 | done() 69 | }).catch(done) 70 | }) 71 | 72 | it("should reject if there is a problem fetching data", done => { 73 | googleapis.ga.get = (params, cb) => { 74 | const error = new Error("i'm an error") 75 | cb(error) 76 | } 77 | 78 | GoogleAnalyticsClient.fetchData({}).catch(error => { 79 | expect(error.message).to.equal("i'm an error") 80 | done() 81 | }).catch(done) 82 | }) 83 | 84 | it("should use the data function if the report is not realtime", done => { 85 | let dataFunctionCalled = false 86 | googleapis.ga.get = (query, cb) => { 87 | dataFunctionCalled = true 88 | cb(null, {}) 89 | } 90 | 91 | GoogleAnalyticsClient.fetchData({}).then(() => { 92 | expect(dataFunctionCalled).to.be.true 93 | done() 94 | }).catch(done) 95 | }) 96 | 97 | it("should use the realtime function if the report is not realtime", done => { 98 | let realtimeFunctionCalled = false 99 | googleapis.realtime.get = (query, cb) => { 100 | realtimeFunctionCalled = true 101 | cb(null, {}) 102 | } 103 | 104 | GoogleAnalyticsClient.fetchData({ realtime: true }).then(() => { 105 | expect(realtimeFunctionCalled).to.be.true 106 | done() 107 | }).catch(done) 108 | }) 109 | }) 110 | }) 111 | -------------------------------------------------------------------------------- /test/publish/s3.test.js: -------------------------------------------------------------------------------- 1 | const expect = require("chai").expect 2 | const proxyquire = require("proxyquire") 3 | const resultsFixture = require("../support/fixtures/results") 4 | 5 | const S3Mock = function() {} 6 | S3Mock.mockedPutObject = () => {} 7 | S3Mock.prototype.putObject = function() { 8 | return S3Mock.mockedPutObject.apply(this, arguments) 9 | } 10 | 11 | const zlibMock = {} 12 | 13 | const S3Publisher = proxyquire("../../src/publish/s3", { 14 | "aws-sdk": { S3: S3Mock }, 15 | "zlib": zlibMock, 16 | "../config": { 17 | aws: { 18 | bucket: "test-bucket", 19 | cache: 60, 20 | path: "path/to/data" 21 | }, 22 | }, 23 | }) 24 | 25 | describe("S3Publisher", () => { 26 | let report 27 | let results 28 | 29 | beforeEach(() => { 30 | results = Object.assign({}, resultsFixture) 31 | report = { name: results.name } 32 | S3Mock.mockedPutObject = () => ({ promise: () => Promise.resolve() }) 33 | zlibMock.gzip = (data, cb) => cb(null, data) 34 | }) 35 | 36 | it("should publish compressed JSON results to the S3 bucket", done => { 37 | report.name = "test-report" 38 | 39 | let s3PutObjectCalled = false 40 | let gzipCalled = false 41 | 42 | S3Mock.mockedPutObject = (options) => { 43 | s3PutObjectCalled = true 44 | 45 | expect(options.Key).to.equal("path/to/data/test-report.json") 46 | expect(options.Bucket).to.equal("test-bucket") 47 | expect(options.ContentType).to.equal("application/json") 48 | expect(options.ContentEncoding).to.equal("gzip") 49 | expect(options.ACL).to.equal("public-read") 50 | expect(options.CacheControl).to.equal("max-age=60") 51 | expect(options.Body).to.equal("compressed data") 52 | 53 | return { promise: () => Promise.resolve() } 54 | } 55 | zlibMock.gzip = (data, cb) => { 56 | gzipCalled = true 57 | cb(null, "compressed data") 58 | } 59 | 60 | S3Publisher.publish(report, `${results}`, { format: "json" }).then(() => { 61 | expect(s3PutObjectCalled).to.equal(true) 62 | expect(gzipCalled).to.equal(true) 63 | done() 64 | }).catch(done) 65 | }) 66 | 67 | it("should publish compressed CSV results to the S3 bucket", done => { 68 | report.name = "test-report" 69 | 70 | let s3PutObjectCalled = false 71 | let gzipCalled = false 72 | 73 | S3Mock.mockedPutObject = (options) => { 74 | s3PutObjectCalled = true 75 | 76 | expect(options.Key).to.equal("path/to/data/test-report.csv") 77 | expect(options.Bucket).to.equal("test-bucket") 78 | expect(options.ContentType).to.equal("text/csv") 79 | expect(options.ContentEncoding).to.equal("gzip") 80 | expect(options.ACL).to.equal("public-read") 81 | expect(options.CacheControl).to.equal("max-age=60") 82 | expect(options.Body).to.equal("compressed data") 83 | 84 | return { promise: () => Promise.resolve() } 85 | } 86 | zlibMock.gzip = (data, cb) => { 87 | gzipCalled = true 88 | cb(null, "compressed data") 89 | } 90 | 91 | S3Publisher.publish(report, `${results}`, { format: "csv" }).then(() => { 92 | expect(s3PutObjectCalled).to.equal(true) 93 | expect(gzipCalled).to.equal(true) 94 | done() 95 | }).catch(done) 96 | }) 97 | 98 | it("should reject if there is an error uploading the data", done => { 99 | S3Mock.mockedPutObject = () => ({ 100 | promise: () => Promise.reject(new Error("test s3 error")) 101 | }) 102 | 103 | S3Publisher.publish(report, `${results}`, { format: "json" }).catch(err => { 104 | expect(err.message).to.equal("test s3 error") 105 | done() 106 | }).catch(done) 107 | }) 108 | 109 | it("should reject if there is an error compressing the data", done => { 110 | zlibMock.gzip = (data, cb) => cb(new Error("test zlib error")) 111 | 112 | S3Publisher.publish(report.name, `${results}`, { format: "json" }).catch(err => { 113 | expect(err.message).to.equal("test zlib error") 114 | done() 115 | }).catch(done) 116 | }) 117 | }) 118 | -------------------------------------------------------------------------------- /src/process-results/result-totals-calculator.js: -------------------------------------------------------------------------------- 1 | const calculateTotals = (result) => { 2 | if (result.data.length === 0) { 3 | return result 4 | } 5 | 6 | let totals = {} 7 | 8 | // Sum up simple columns 9 | if ("users" in result.data[0]) { 10 | totals.users = _sumColumn({ column: "users", result }) 11 | } 12 | if ("visits" in result.data[0]) { 13 | totals.visits = _sumColumn({ column: "visits", result }) 14 | } 15 | 16 | // Sum up categories 17 | if (result.name.match(/^device_model/)) { 18 | totals.device_models = _sumVisitsByColumn({ 19 | column: "mobile_device", 20 | result, 21 | }) 22 | } 23 | if (result.name.match(/^language/)) { 24 | totals.languages = _sumVisitsByColumn({ 25 | column: "language", 26 | result, 27 | }) 28 | } 29 | if (result.name.match(/^devices/)) { 30 | totals.devices = _sumVisitsByColumn({ 31 | column: "device", 32 | result, 33 | }) 34 | } 35 | if (result.name == "screen-size") { 36 | totals.screen_resolution = _sumVisitsByColumn({ 37 | column: "screen_resolution", 38 | result, 39 | }) 40 | } 41 | if (result.name === "os") { 42 | totals.os = _sumVisitsByColumn({ 43 | column: "os", 44 | result, 45 | }) 46 | } 47 | if (result.name === "windows") { 48 | totals.os_version = _sumVisitsByColumn({ 49 | column: "os_version", 50 | result, 51 | }) 52 | } 53 | if (result.name === "browsers") { 54 | totals.browser = _sumVisitsByColumn({ 55 | column: "browser", 56 | result, 57 | }) 58 | } 59 | if (result.name === "ie") { 60 | totals.ie_version = _sumVisitsByColumn({ 61 | column: "browser_version", 62 | result, 63 | }) 64 | } 65 | 66 | // Sum up totals with 2 levels of hashes 67 | if (result.name === "os-browsers") { 68 | totals.by_os = _sumVisitsByCategoryWithDimension({ 69 | column: "os", 70 | dimension: "browser", 71 | result, 72 | }) 73 | totals.by_browsers = _sumVisitsByCategoryWithDimension({ 74 | column: "browser", 75 | dimension: "os", 76 | result, 77 | }) 78 | } 79 | if (result.name === "windows-ie") { 80 | totals.by_windows = _sumVisitsByCategoryWithDimension({ 81 | column: "os_version", 82 | dimension: "browser_version", 83 | result, 84 | }) 85 | totals.by_ie = _sumVisitsByCategoryWithDimension({ 86 | column: "browser_version", 87 | dimension: "os_version", 88 | result, 89 | }) 90 | } 91 | if (result.name === "windows-browsers") { 92 | totals.by_windows = _sumVisitsByCategoryWithDimension({ 93 | column: "os_version", 94 | dimension: "browser", 95 | result, 96 | }) 97 | totals.by_browsers = _sumVisitsByCategoryWithDimension({ 98 | column: "browser", 99 | dimension: "os_version", 100 | result, 101 | }) 102 | } 103 | 104 | // Set the start and end date 105 | if (result.data[0].data) { 106 | // Occasionally we'll get bogus start dates 107 | if (result.date[0].date === "(other)") { 108 | totals.start_date = result.data[1].date 109 | } else { 110 | totals.start_date = result.data[0].date 111 | } 112 | totals.end_date = result.data[result.data.length-1].date 113 | } 114 | 115 | return totals 116 | } 117 | 118 | const _sumColumn = ({ result, column }) => { 119 | return result.data.reduce((total, row) => { 120 | return parseInt(row[column]) + total 121 | }, 0) 122 | } 123 | 124 | const _sumVisitsByColumn = ({ result, column }) => { 125 | return result.data.reduce((categories, row) => { 126 | const category = row[column] 127 | const visits = parseInt(row.visits) 128 | categories[category] = (categories[category] || 0) + visits 129 | return categories 130 | }, {}) 131 | } 132 | 133 | const _sumVisitsByCategoryWithDimension = ({ result, column, dimension }) => { 134 | return result.data.reduce((categories, row) => { 135 | const parentCategory = row[column] 136 | const childCategory = row[dimension] 137 | const visits = parseInt(row.visits) 138 | 139 | categories[parentCategory] = categories[parentCategory] || {} 140 | 141 | const newTotal = (categories[parentCategory][childCategory] || 0) + visits 142 | categories[parentCategory][childCategory] = newTotal 143 | 144 | return categories 145 | }, {}) 146 | } 147 | 148 | module.exports = { calculateTotals } 149 | -------------------------------------------------------------------------------- /src/process-results/ga-data-processor.js: -------------------------------------------------------------------------------- 1 | const config = require("../config") 2 | const ResultTotalsCalculator = require("./result-totals-calculator") 3 | 4 | const processData = (report, data) => { 5 | let result = _initializeResult({ report, data }) 6 | 7 | // If you use a filter that results in no data, you get null 8 | // back from google and need to protect against it. 9 | if (!data || !data.rows) { 10 | return result; 11 | } 12 | 13 | // Some reports may decide to cut fields from the output. 14 | if (report.cut) { 15 | data = _removeColumnFromData({ column: report.cut, data }) 16 | } 17 | 18 | // Remove data points that are below the threshold if one exists 19 | if (report.threshold) { 20 | data = _filterRowsBelowThreshold({ threshold: report.threshold, data }) 21 | } 22 | 23 | // Process each row 24 | result.data = data.rows.map(row => { 25 | return _processRow({ row, report, data }) 26 | }) 27 | 28 | result.totals = ResultTotalsCalculator.calculateTotals(result) 29 | 30 | return result; 31 | } 32 | 33 | const _fieldNameForColumnIndex = ({ index, data }) => { 34 | const name = data.columnHeaders[index].name 35 | return _mapping[name] || name 36 | } 37 | 38 | const _filterRowsBelowThreshold = ({ threshold, data }) => { 39 | data = Object.assign({}, data) 40 | 41 | const thresholdIndex = data.columnHeaders.findIndex(header => { 42 | return header.name === threshold.field 43 | }) 44 | const thresholdValue = parseInt(threshold.value) 45 | 46 | data.rows = data.rows.filter(row => { 47 | return row[thresholdIndex] >= thresholdValue 48 | }) 49 | 50 | return data 51 | } 52 | 53 | const _formatDate = (date) => { 54 | if (date == "(other)") { 55 | return date 56 | } 57 | return [date.substr(0,4), date.substr(4, 2), date.substr(6, 2)].join("-") 58 | } 59 | 60 | const _initializeResult = ({ report, data }) => ({ 61 | name: report.name, 62 | sampling: { 63 | containsSampledData: data.containsSampledData, 64 | sampleSize: data.sampleSize, 65 | sampleSpace: data.sampleSpace 66 | }, 67 | query: ((query) => { 68 | query = Object.assign({}, query) 69 | delete query.ids 70 | return query 71 | })(data.query), 72 | meta: report.meta, 73 | data: [], 74 | totals: {}, 75 | taken_at: new Date(), 76 | }) 77 | 78 | const _processRow = ({ row, data, report }) => { 79 | const point = {} 80 | 81 | row.forEach((rowElement, index) => { 82 | const field = _fieldNameForColumnIndex({ index, data }) 83 | let value = rowElement 84 | if (field === "date") { 85 | value = _formatDate(value) 86 | } 87 | point[field] = value 88 | }) 89 | 90 | if (config.account.hostname && !('domain' in point)) { 91 | point.domain = config.account.hostname 92 | } 93 | 94 | return point 95 | } 96 | 97 | const _removeColumnFromData = ({ column, data }) => { 98 | data = Object.assign(data) 99 | 100 | const columnIndex = data.columnHeaders.findIndex(header => { 101 | return header.name === column 102 | }) 103 | 104 | data.columnHeaders.splice(columnIndex, 1) 105 | data.rows.forEach(row => { 106 | row.splice(columnIndex, 1) 107 | }) 108 | 109 | return data 110 | } 111 | 112 | const _mapping = { 113 | "ga:date": "date", 114 | "ga:hour": "hour", 115 | "rt:activeUsers": "active_visitors", 116 | "rt:pagePath": "page", 117 | "rt:pageTitle": "page_title", 118 | "ga:sessions": "visits", 119 | "ga:deviceCategory": "device", 120 | "ga:operatingSystem": "os", 121 | "ga:operatingSystemVersion": "os_version", 122 | "ga:hostname": "domain", 123 | "ga:browser" : 'browser', 124 | "ga:browserVersion" : "browser_version", 125 | "ga:source": "source", 126 | "ga:pagePath": "page", 127 | "ga:pageTitle": "page_title", 128 | "ga:pageviews": "visits", 129 | "ga:country": "country", 130 | "ga:city": 'city', 131 | "ga:eventLabel": "event_label", 132 | "ga:totalEvents": "total_events", 133 | "ga:landingPagePath": "landing_page", 134 | "ga:exitPagePath": "exit_page", 135 | "ga:source": "source", 136 | "ga:hasSocialSourceReferral": "has_social_referral", 137 | "ga:referralPath": "referral_path", 138 | "ga:pageviews": "pageviews", 139 | "ga:users": "users", 140 | "ga:pageviewsPerSession": "pageviews_per_session", 141 | "ga:avgSessionDuration": "avg_session_duration", 142 | "ga:exits": "exits", 143 | "ga:language": "language", 144 | "ga:screenResolution": "screen_resolution", 145 | "ga:mobileDeviceModel": "mobile_device", 146 | "rt:country": "country", 147 | "rt:city": "city", 148 | "rt:totalEvents": "total_events", 149 | "rt:eventLabel": "event_label" 150 | } 151 | 152 | module.exports = { processData } 153 | -------------------------------------------------------------------------------- /test/google-analytics/query-authorizer.test.js: -------------------------------------------------------------------------------- 1 | const expect = require("chai").expect 2 | const proxyquire = require("proxyquire") 3 | const googleAPIsMock = require("../support/mocks/googleapis-auth") 4 | 5 | proxyquire.noCallThru() 6 | 7 | const config = {} 8 | const googleapis = {} 9 | const GoogleAnalyticsCredentialLoader = { 10 | loadCredentials: () => ({ 11 | email: "next_email@example.com", 12 | key: "Shhh, this is the next secret", 13 | }) 14 | } 15 | 16 | const GoogleAnalyticsQueryAuthorizer = proxyquire("../../src/google-analytics/query-authorizer", { 17 | "../config": config, 18 | "./credential-loader": GoogleAnalyticsCredentialLoader, 19 | googleapis, 20 | }) 21 | 22 | describe("GoogleAnalyticsQueryAuthorizer", () => { 23 | describe(".authorizeQuery(query)", () => { 24 | beforeEach(() => { 25 | Object.assign(googleapis, googleAPIsMock()) 26 | config.email = "hello@example.com" 27 | config.key = "123abc" 28 | config.key_file = undefined 29 | }) 30 | 31 | it("should resolve a query with the auth prop set to an authorized JWT", done => { 32 | query = { 33 | "abc": 123 34 | } 35 | 36 | GoogleAnalyticsQueryAuthorizer.authorizeQuery(query).then(query => { 37 | expect(query.abc).to.equal(123) 38 | expect(query.auth).to.not.be.undefined 39 | expect(query.auth).to.be.an.instanceof(googleapis.auth.JWT) 40 | done() 41 | }).catch(done) 42 | }) 43 | 44 | it("should create a JWT with the key and email in the config if one exists", done => { 45 | config.email = "test@example.com" 46 | config.key = "Shh, this is a secret" 47 | 48 | GoogleAnalyticsQueryAuthorizer.authorizeQuery({}).then(query => { 49 | expect(query.auth.initArguments[0]).to.equal("test@example.com") 50 | expect(query.auth.initArguments[2]).to.equal("Shh, this is a secret") 51 | done() 52 | }).catch(done) 53 | }) 54 | 55 | it("should create a JWT from the keyfile and the email in the config if one exists", done => { 56 | config.email = "test@example.com" 57 | config.key = undefined 58 | config.key_file = "./test/support/fixtures/secret_key.pem" 59 | 60 | GoogleAnalyticsQueryAuthorizer.authorizeQuery({}).then(query => { 61 | expect(query.auth.initArguments[0]).to.equal("test@example.com") 62 | expect(query.auth.initArguments[2]).to.equal("pem-key-file-not-actually-a-secret-key") 63 | done() 64 | }).catch(done) 65 | }) 66 | 67 | it("should create a JWT from the JSON keyfile in the config if one exists", done => { 68 | config.key = undefined 69 | config.key_file = "./test/support/fixtures/secret_key.json" 70 | 71 | GoogleAnalyticsQueryAuthorizer.authorizeQuery({}).then(query => { 72 | expect(query.auth.initArguments[0]).to.equal("json_test_email@example.com") 73 | expect(query.auth.initArguments[2]).to.equal("json-key-file-not-actually-a-secret-key") 74 | done() 75 | }).catch(done) 76 | }) 77 | 78 | it("should create a JWT with credentials from calling GoogleAnalyticsCredentialLoader for analytics credentials in the config", done => { 79 | config.key = undefined 80 | config.analytics_credentials = "[{}]" // overriden by proxyquire 81 | 82 | GoogleAnalyticsQueryAuthorizer.authorizeQuery({}).then(query => { 83 | expect(query.auth.initArguments[0]).to.equal("next_email@example.com") 84 | expect(query.auth.initArguments[2]).to.equal("Shhh, this is the next secret") 85 | done() 86 | }).catch(done) 87 | }) 88 | 89 | it("should create a JWT with the proper scopes", done => { 90 | GoogleAnalyticsQueryAuthorizer.authorizeQuery({}).then(query => { 91 | expect(query.auth.initArguments[3]).to.deep.equal([ 92 | "https://www.googleapis.com/auth/analytics.readonly" 93 | ]) 94 | done() 95 | }).catch(done) 96 | }) 97 | 98 | it("should authorize the JWT and resolve if it is valid", done => { 99 | let jwtAuthorized = false 100 | googleapis.auth.JWT.prototype.authorize = (callback) => { 101 | jwtAuthorized = true 102 | callback(null, {}) 103 | } 104 | 105 | GoogleAnalyticsQueryAuthorizer.authorizeQuery({}).then(query => { 106 | expect(jwtAuthorized).to.equal(true) 107 | done() 108 | }).catch(done) 109 | }) 110 | 111 | it("should authorize the JWT and reject if it is invalid", done => { 112 | let jwtAuthorized = false 113 | googleapis.auth.JWT.prototype.authorize = (callback) => { 114 | jwtAuthorized = true 115 | callback(new Error("Failed to authorize")) 116 | } 117 | 118 | GoogleAnalyticsQueryAuthorizer.authorizeQuery({}).catch(err => { 119 | expect(jwtAuthorized).to.equal(true) 120 | expect(err.message).to.equal("Failed to authorize") 121 | done() 122 | }).catch(done) 123 | }) 124 | }) 125 | }) 126 | -------------------------------------------------------------------------------- /test/process-results/ga-data-processor.test.js: -------------------------------------------------------------------------------- 1 | const expect = require("chai").expect 2 | const proxyquire = require("proxyquire") 3 | const reportFixture = require("../support/fixtures/report") 4 | const dataFixture = require("../support/fixtures/data") 5 | const dataWithHostnameFixture = require("../support/fixtures/data_with_hostname") 6 | 7 | proxyquire.noCallThru() 8 | 9 | const config = {} 10 | 11 | const GoogleAnalyticsDataProcessor = proxyquire("../../src/process-results/ga-data-processor", { 12 | "../config": config, 13 | }) 14 | 15 | describe("GoogleAnalyticsDataProcessor", () => { 16 | describe(".processData(report, data)", () => { 17 | let report 18 | let data 19 | 20 | beforeEach(() => { 21 | report = Object.assign({}, reportFixture) 22 | data = Object.assign({}, dataFixture) 23 | config.account = { 24 | hostname: "" 25 | } 26 | }) 27 | 28 | it("should return results with the correct props", () => { 29 | const result = GoogleAnalyticsDataProcessor.processData(report, data) 30 | expect(result.name).to.be.a("string") 31 | expect(result.query).be.an("object") 32 | expect(result.meta).be.an("object") 33 | expect(result.data).be.an("array") 34 | expect(result.totals).be.an("object") 35 | expect(result.totals).be.an("object") 36 | expect(result.taken_at).be.a("date") 37 | }) 38 | 39 | it("should return results with an empty data array if data is undefined or has no rows", () => { 40 | data.rows = [] 41 | expect(GoogleAnalyticsDataProcessor.processData(report, data).data).to.be.empty 42 | data.rows = undefined 43 | expect(GoogleAnalyticsDataProcessor.processData(report, data).data).to.be.empty 44 | }) 45 | 46 | it("should delete the query ids for the GA response", () => { 47 | const result = GoogleAnalyticsDataProcessor.processData(report, data) 48 | expect(result.query).to.not.have.property("ids") 49 | }) 50 | 51 | it("should map data from GA keys to DAP keys", () => { 52 | data.columnHeaders = [ 53 | { name: "ga:date" }, { name: "ga:browser"}, { name: "ga:city" } 54 | ] 55 | data.rows = [["20170130", "chrome", "Baton Rouge, La"]] 56 | 57 | const result = GoogleAnalyticsDataProcessor.processData(report, data) 58 | expect(Object.keys(result.data[0])).to.deep.equal(["date", "browser", "city"]) 59 | }) 60 | 61 | it("should format dates", () => { 62 | data.columnHeaders = [{ name: 'ga:date' }] 63 | data.rows = [[ "20170130" ]] 64 | 65 | const result = GoogleAnalyticsDataProcessor.processData(report, data) 66 | expect(result.data[0].date).to.equal("2017-01-30") 67 | }) 68 | 69 | it("should filter rows that don't meet the threshold if a threshold is provided", () => { 70 | report.threshold = { 71 | field: "unmapped_column", 72 | value: "10", 73 | } 74 | data.columnHeaders = [{ name: "unmapped_column" }] 75 | data.rows = [[20], [5], [15]] 76 | 77 | const result = GoogleAnalyticsDataProcessor.processData(report, data) 78 | expect(result.data).to.have.length(2) 79 | expect(result.data.map(row => row.unmapped_column)).to.deep.equal([20, 15]) 80 | }) 81 | 82 | it("should remove dimensions that are specified by the cut prop", () => { 83 | report.cut = "unmapped_column" 84 | data.columnHeaders = [{ name: "ga:hostname" }, { name: "unmapped_column" }] 85 | data.rows = [["www.example.gov", 10000000]] 86 | 87 | const result = GoogleAnalyticsDataProcessor.processData(report, data) 88 | expect(result.data[0].unmapped_column).to.be.undefined 89 | }) 90 | 91 | it("should add a hostname to realtime data if a hostname is specified by the config", () => { 92 | report.realtime = true 93 | config.account.hostname = "www.example.gov" 94 | 95 | const result = GoogleAnalyticsDataProcessor.processData(report, data) 96 | expect(result.data[0].domain).to.equal("www.example.gov") 97 | }) 98 | 99 | it("should not overwrite the domain with a hostname from the config", () => { 100 | let dataWithHostname 101 | dataWithHostname = Object.assign({}, dataWithHostnameFixture) 102 | report.realtime = true 103 | config.account.hostname = "www.example.gov" 104 | 105 | const result = GoogleAnalyticsDataProcessor.processData(report, dataWithHostname) 106 | expect(result.data[0].domain).to.equal("www.example0.com") 107 | }) 108 | 109 | it("should set use ResultTotalsCalculator to calculate the totals", () => { 110 | const calculateTotals = (result) => { 111 | expect(result.name).to.equal(report.name) 112 | expect(result.data).to.be.an("array") 113 | return { "visits": 1234 } 114 | } 115 | const GoogleAnalyticsDataProcessor = proxyquire("../../src/process-results/ga-data-processor", { 116 | "./config": config, 117 | "./result-totals-calculator": { calculateTotals }, 118 | }) 119 | 120 | const result = GoogleAnalyticsDataProcessor.processData(report, data) 121 | expect(result.totals).to.deep.equal({ "visits": 1234 }) 122 | }) 123 | }) 124 | }) 125 | -------------------------------------------------------------------------------- /deploy/api.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export ANALYTICS_REPORTS_PATH=reports/api.json 4 | 5 | # Gov Wide 6 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 7 | 8 | # Department of Education 9 | source $ANALYTICS_ROOT_PATH/deploy/envs/education.env 10 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 11 | 12 | # Department of Veterans Affairs 13 | source $ANALYTICS_ROOT_PATH/deploy/envs/veterans-affairs.env 14 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 15 | 16 | # National Aeronautics and Space Administration 17 | source $ANALYTICS_ROOT_PATH/deploy/envs/national-aeronautics-space-administration.env 18 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 19 | 20 | # Department of Justice 21 | source $ANALYTICS_ROOT_PATH/deploy/envs/justice.env 22 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 23 | 24 | # Department of Commerce 25 | source $ANALYTICS_ROOT_PATH/deploy/envs/commerce.env 26 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 27 | 28 | # Environmental Protection Agency 29 | source $ANALYTICS_ROOT_PATH/deploy/envs/environmental-protection-agency.env 30 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 31 | 32 | # Small Business Administration 33 | source $ANALYTICS_ROOT_PATH/deploy/envs/small-business-administration.env 34 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 35 | 36 | # Department of Energy 37 | source $ANALYTICS_ROOT_PATH/deploy/envs/energy.env 38 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 39 | 40 | # Department of the Interior 41 | source $ANALYTICS_ROOT_PATH/deploy/envs/interior.env 42 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 43 | 44 | # National Archives and Records Administration 45 | source $ANALYTICS_ROOT_PATH/deploy/envs/national-archives-records-administration.env 46 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 47 | 48 | # Department of Agriculture 49 | source $ANALYTICS_ROOT_PATH/deploy/envs/agriculture.env 50 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 51 | 52 | # Department of Defense 53 | source $ANALYTICS_ROOT_PATH/deploy/envs/defense.env 54 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 55 | 56 | # Department of Health and Human Services 57 | source $ANALYTICS_ROOT_PATH/deploy/envs/health-human-services.env 58 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 59 | 60 | # Department of Housing and Urban Development 61 | source $ANALYTICS_ROOT_PATH/deploy/envs/housing-urban-development.env 62 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 63 | 64 | # Department of Homeland Security 65 | source $ANALYTICS_ROOT_PATH/deploy/envs/homeland-security.env 66 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 67 | 68 | # Department of Labor 69 | source $ANALYTICS_ROOT_PATH/deploy/envs/labor.env 70 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 71 | 72 | # Department of State 73 | source $ANALYTICS_ROOT_PATH/deploy/envs/state.env 74 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 75 | 76 | # Department of Transportation 77 | source $ANALYTICS_ROOT_PATH/deploy/envs/transportation.env 78 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 79 | 80 | # Department of the Treasury 81 | source $ANALYTICS_ROOT_PATH/deploy/envs/treasury.env 82 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 83 | 84 | # Agency for International Development 85 | source $ANALYTICS_ROOT_PATH/deploy/envs/agency-international-development.env 86 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 87 | 88 | # General Services Administration 89 | source $ANALYTICS_ROOT_PATH/deploy/envs/general-services-administration.env 90 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 91 | 92 | # National Science Foundation 93 | source $ANALYTICS_ROOT_PATH/deploy/envs/national-science-foundation.env 94 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 95 | 96 | # Nuclear Regulatory Commission 97 | source $ANALYTICS_ROOT_PATH/deploy/envs/nuclear-regulatory-commission.env 98 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 99 | 100 | # Office of Personnel Management 101 | source $ANALYTICS_ROOT_PATH/deploy/envs/office-personnel-management.env 102 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 103 | 104 | # Social Security Administration 105 | source $ANALYTICS_ROOT_PATH/deploy/envs/social-security-administration.env 106 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 107 | 108 | # Postal Service 109 | source $ANALYTICS_ROOT_PATH/deploy/envs/postal-service.env 110 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 111 | 112 | # Executive Office of the President 113 | source $ANALYTICS_ROOT_PATH/deploy/envs/executive-office-president.env 114 | $ANALYTICS_ROOT_PATH/bin/analytics --verbose --write-to-database --output /tmp 115 | -------------------------------------------------------------------------------- /deploy/hourly.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Government Wide 4 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 5 | 6 | # Department of Education 7 | source $ANALYTICS_ROOT_PATH/deploy/envs/education.env 8 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 9 | 10 | # National Aeronautics and Space Administration 11 | source $ANALYTICS_ROOT_PATH/deploy/envs/national-aeronautics-space-administration.env 12 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 13 | 14 | # Department of Justice 15 | source $ANALYTICS_ROOT_PATH/deploy/envs/justice.env 16 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 17 | 18 | # Department of Veterans Affairs 19 | source $ANALYTICS_ROOT_PATH/deploy/envs/veterans-affairs.env 20 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 21 | 22 | # Department of Commerce 23 | source $ANALYTICS_ROOT_PATH/deploy/envs/commerce.env 24 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 25 | 26 | # Environmental Protection Agency 27 | source $ANALYTICS_ROOT_PATH/deploy/envs/environmental-protection-agency.env 28 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 29 | 30 | # Small Business Administration 31 | source $ANALYTICS_ROOT_PATH/deploy/envs/small-business-administration.env 32 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 33 | 34 | # Department of Energy 35 | source $ANALYTICS_ROOT_PATH/deploy/envs/energy.env 36 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 37 | 38 | # Department of the Interior 39 | source $ANALYTICS_ROOT_PATH/deploy/envs/interior.env 40 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 41 | 42 | # National Archives and Records Administration 43 | source $ANALYTICS_ROOT_PATH/deploy/envs/national-archives-records-administration.env 44 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 45 | 46 | # Department of Agriculture 47 | source $ANALYTICS_ROOT_PATH/deploy/envs/agriculture.env 48 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 49 | 50 | # Department of Defense 51 | source $ANALYTICS_ROOT_PATH/deploy/envs/defense.env 52 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 53 | 54 | # Department of Health and Human Services 55 | source $ANALYTICS_ROOT_PATH/deploy/envs/health-human-services.env 56 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 57 | 58 | # Department of Housing and Urban Development 59 | source $ANALYTICS_ROOT_PATH/deploy/envs/housing-urban-development.env 60 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 61 | 62 | # Department of Homeland Security 63 | source $ANALYTICS_ROOT_PATH/deploy/envs/homeland-security.env 64 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 65 | 66 | # Department of Labor 67 | source $ANALYTICS_ROOT_PATH/deploy/envs/labor.env 68 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 69 | 70 | # Department of State 71 | source $ANALYTICS_ROOT_PATH/deploy/envs/state.env 72 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 73 | 74 | # Department of Transportation 75 | source $ANALYTICS_ROOT_PATH/deploy/envs/transportation.env 76 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 77 | 78 | # Department of the Treasury 79 | source $ANALYTICS_ROOT_PATH/deploy/envs/treasury.env 80 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 81 | 82 | # Agency for International Development 83 | source $ANALYTICS_ROOT_PATH/deploy/envs/agency-international-development.env 84 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 85 | 86 | # General Services Administration 87 | source $ANALYTICS_ROOT_PATH/deploy/envs/general-services-administration.env 88 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 89 | 90 | # National Science Foundation 91 | source $ANALYTICS_ROOT_PATH/deploy/envs/national-science-foundation.env 92 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 93 | 94 | # Nuclear Regulatory Commission 95 | source $ANALYTICS_ROOT_PATH/deploy/envs/nuclear-regulatory-commission.env 96 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 97 | 98 | # Office of Personnel Management 99 | source $ANALYTICS_ROOT_PATH/deploy/envs/office-personnel-management.env 100 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 101 | 102 | # Social Security Administration 103 | source $ANALYTICS_ROOT_PATH/deploy/envs/social-security-administration.env 104 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 105 | 106 | # Postal Service 107 | source $ANALYTICS_ROOT_PATH/deploy/envs/postal-service.env 108 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 109 | 110 | # Executive Office of the President 111 | source $ANALYTICS_ROOT_PATH/deploy/envs/executive-office-president.env 112 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=hourly --slim --verbose 113 | -------------------------------------------------------------------------------- /test/publish/postgres.test.js: -------------------------------------------------------------------------------- 1 | const { ANALYTICS_DATA_TABLE_NAME } = require("../../src/publish/postgres") 2 | 3 | const expect = require("chai").expect 4 | const knex = require("knex") 5 | const proxyquire = require("proxyquire") 6 | const database = require("../support/database") 7 | const resultsFixture = require("../support/fixtures/results") 8 | 9 | proxyquire.noCallThru() 10 | 11 | const config = { 12 | postgres: database.connection, 13 | timezone: "US/Eastern", 14 | } 15 | 16 | const PostgresPublisher = proxyquire("../../src/publish/postgres", { 17 | "../config": config, 18 | }) 19 | 20 | const databaseClient = knex({ client: "pg", connection: database.connection }) 21 | 22 | describe("PostgresPublisher", () => { 23 | describe(".publish(results)", () => { 24 | let results 25 | 26 | beforeEach(done => { 27 | results = Object.assign({}, resultsFixture) 28 | 29 | database.resetSchema().then(() => { 30 | done() 31 | }).catch(done) 32 | }) 33 | 34 | it("should insert a record for each results.data element", done => { 35 | results.name = "report-name" 36 | results.data = [ 37 | { 38 | date: "2017-02-11", 39 | name: "abc", 40 | }, 41 | { 42 | date: "2017-02-12", 43 | name: "def", 44 | }, 45 | ] 46 | 47 | PostgresPublisher.publish(results).then(() => { 48 | return databaseClient(ANALYTICS_DATA_TABLE_NAME).orderBy("date", "asc").select() 49 | }).then(rows => { 50 | expect(rows).to.have.length(2) 51 | rows.forEach((row, index) => { 52 | const data = results.data[index] 53 | expect(row.report_name).to.equal("report-name") 54 | expect(row.data.name).to.equal(data.name) 55 | expect(row.date.toISOString()).to.match(RegExp(`^${data.date}`)) 56 | }) 57 | done() 58 | }).catch(done) 59 | }) 60 | 61 | it("should coerce certain values into numbers", done => { 62 | results.name = "report-name" 63 | results.data = [{ 64 | date: "2017-05-15", 65 | name: "abc", 66 | visits: "123", 67 | total_events: "456", 68 | }] 69 | 70 | PostgresPublisher.publish(results).then(() => { 71 | return databaseClient.select().table(ANALYTICS_DATA_TABLE_NAME) 72 | }).then(rows => { 73 | const row = rows[0] 74 | expect(row.data.visits).to.be.a("number") 75 | expect(row.data.visits).to.equal(123) 76 | expect(row.data.total_events).to.be.a("number") 77 | expect(row.data.total_events).to.equal(456) 78 | done() 79 | }).catch(done) 80 | }) 81 | 82 | it("should ignore reports that don't have a ga:date dimension", done => { 83 | results.query = { dimensions: "ga:something,ga:somethingElse" } 84 | 85 | PostgresPublisher.publish(results).then(() => { 86 | return databaseClient.select().table(ANALYTICS_DATA_TABLE_NAME) 87 | }).then(rows => { 88 | expect(rows).to.have.length(0) 89 | done() 90 | }).catch(done) 91 | }) 92 | 93 | it("should ignore data points that have already been inserted", done => { 94 | firstResults = Object.assign({}, results) 95 | secondResults = Object.assign({}, results) 96 | 97 | firstResults.data = [ 98 | { 99 | date: "2017-02-11", 100 | visits: "123", 101 | browser: "Chrome", 102 | }, 103 | { 104 | date: "2017-02-11", 105 | visits: "456", 106 | browser: "Safari" 107 | }, 108 | ] 109 | secondResults.data = [ 110 | { 111 | date: "2017-02-11", 112 | visits: "456", 113 | browser: "Safari", 114 | }, 115 | { 116 | date: "2017-02-11", 117 | visits: "789", 118 | browser: "Internet Explorer" 119 | }, 120 | ] 121 | 122 | PostgresPublisher.publish(firstResults).then(() => { 123 | return PostgresPublisher.publish(secondResults) 124 | }).then(() => { 125 | return databaseClient.select().table(ANALYTICS_DATA_TABLE_NAME) 126 | }).then(rows => { 127 | expect(rows).to.have.length(3) 128 | done() 129 | }).catch(done) 130 | }) 131 | 132 | it("should overwrite existing data points if the number of visits or users has changed", done => { 133 | firstResults = Object.assign({}, results) 134 | secondResults = Object.assign({}, results) 135 | 136 | firstResults.data = [ 137 | { 138 | date: "2017-02-11", 139 | visits: "100", 140 | browser: "Safari", 141 | }, 142 | { 143 | date: "2017-02-11", 144 | total_events: "300", 145 | title: "IRS Form 123", 146 | }, 147 | ] 148 | secondResults.data = [ 149 | { 150 | date: "2017-02-11", 151 | visits: "200", 152 | browser: "Safari", 153 | }, 154 | { 155 | date: "2017-02-11", 156 | total_events: "400", 157 | title: "IRS Form 123", 158 | }, 159 | ] 160 | 161 | PostgresPublisher.publish(firstResults).then(() => { 162 | return PostgresPublisher.publish(secondResults) 163 | }).then(() => { 164 | return databaseClient.select().table(ANALYTICS_DATA_TABLE_NAME) 165 | }).then(rows => { 166 | expect(rows).to.have.length(2) 167 | rows.forEach(row => { 168 | if (row.data.visits) { 169 | expect(row.data.visits).to.equal(200) 170 | } else { 171 | expect(row.data.total_events).to.equal(400) 172 | } 173 | }) 174 | done() 175 | }).catch(done) 176 | }) 177 | 178 | it("should not not insert a record if the date is invalid", done => { 179 | results.data = [ 180 | { 181 | date: "(other)", 182 | visits: "123", 183 | }, 184 | { 185 | date: "2017-02-16", 186 | visits: "456", 187 | }, 188 | ] 189 | 190 | PostgresPublisher.publish(results).then(() => { 191 | return databaseClient.select().table(ANALYTICS_DATA_TABLE_NAME) 192 | }).then(rows => { 193 | expect(rows).to.have.length(1) 194 | expect(rows[0].data.visits).to.equal(456) 195 | done() 196 | }).catch(done) 197 | }) 198 | }) 199 | }) 200 | -------------------------------------------------------------------------------- /deploy/daily.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Government Wide 4 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 5 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 6 | 7 | # Department of Education 8 | source $ANALYTICS_ROOT_PATH/deploy/envs/education.env 9 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 10 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 11 | 12 | # Department of Veterans Affairs 13 | source $ANALYTICS_ROOT_PATH/deploy/envs/veterans-affairs.env 14 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 15 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 16 | 17 | # National Aeronautics and Space Administration 18 | source $ANALYTICS_ROOT_PATH/deploy/envs/national-aeronautics-space-administration.env 19 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 20 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 21 | 22 | # Department of Justice 23 | source $ANALYTICS_ROOT_PATH/deploy/envs/justice.env 24 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 25 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 26 | 27 | # Department of Commerce 28 | source $ANALYTICS_ROOT_PATH/deploy/envs/commerce.env 29 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 30 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 31 | 32 | # Environmental Protection Agency 33 | source $ANALYTICS_ROOT_PATH/deploy/envs/environmental-protection-agency.env 34 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 35 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 36 | 37 | # Small Business Administration 38 | source $ANALYTICS_ROOT_PATH/deploy/envs/small-business-administration.env 39 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 40 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 41 | 42 | # Department of Energy 43 | source $ANALYTICS_ROOT_PATH/deploy/envs/energy.env 44 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 45 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 46 | 47 | # Department of the Interior 48 | source $ANALYTICS_ROOT_PATH/deploy/envs/interior.env 49 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 50 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 51 | 52 | # National Archives and Records Administration 53 | source $ANALYTICS_ROOT_PATH/deploy/envs/national-archives-records-administration.env 54 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 55 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 56 | 57 | # Department of Agriculture 58 | source $ANALYTICS_ROOT_PATH/deploy/envs/agriculture.env 59 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 60 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 61 | 62 | # Department of Defense 63 | source $ANALYTICS_ROOT_PATH/deploy/envs/defense.env 64 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 65 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 66 | 67 | # Department of Health and Human Services 68 | source $ANALYTICS_ROOT_PATH/deploy/envs/health-human-services.env 69 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 70 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 71 | 72 | # Department of Housing and Urban Development 73 | source $ANALYTICS_ROOT_PATH/deploy/envs/housing-urban-development.env 74 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 75 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 76 | 77 | # Department of Homeland Security 78 | source $ANALYTICS_ROOT_PATH/deploy/envs/homeland-security.env 79 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 80 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 81 | 82 | # Department of Labor 83 | source $ANALYTICS_ROOT_PATH/deploy/envs/labor.env 84 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 85 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 86 | 87 | # Department of State 88 | source $ANALYTICS_ROOT_PATH/deploy/envs/state.env 89 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 90 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 91 | 92 | # Department of Transportation 93 | source $ANALYTICS_ROOT_PATH/deploy/envs/transportation.env 94 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 95 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 96 | 97 | # Department of the Treasury 98 | source $ANALYTICS_ROOT_PATH/deploy/envs/treasury.env 99 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 100 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 101 | 102 | # Agency for International Development 103 | source $ANALYTICS_ROOT_PATH/deploy/envs/agency-international-development.env 104 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 105 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 106 | 107 | # General Services Administration 108 | source $ANALYTICS_ROOT_PATH/deploy/envs/general-services-administration.env 109 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 110 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 111 | 112 | # National Science Foundation 113 | source $ANALYTICS_ROOT_PATH/deploy/envs/national-science-foundation.env 114 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 115 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 116 | 117 | # Nuclear Regulatory Commission 118 | source $ANALYTICS_ROOT_PATH/deploy/envs/nuclear-regulatory-commission.env 119 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 120 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 121 | 122 | # Office of Personnel Management 123 | source $ANALYTICS_ROOT_PATH/deploy/envs/office-personnel-management.env 124 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 125 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 126 | 127 | # Social Security Administration 128 | source $ANALYTICS_ROOT_PATH/deploy/envs/social-security-administration.env 129 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 130 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 131 | 132 | # Postal Service 133 | source $ANALYTICS_ROOT_PATH/deploy/envs/postal-service.env 134 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 135 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 136 | 137 | # Executive Office of the President 138 | source $ANALYTICS_ROOT_PATH/deploy/envs/executive-office-president.env 139 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose 140 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=daily --slim --verbose --csv 141 | -------------------------------------------------------------------------------- /deploy/realtime.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Government Wide 4 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 5 | # we want just one realtime report in CSV, hardcoded for now to save on API requests 6 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 7 | 8 | # Department of Education 9 | source $ANALYTICS_ROOT_PATH/deploy/envs/education.env 10 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 11 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 12 | 13 | # Department of Veterans Affairs 14 | source $ANALYTICS_ROOT_PATH/deploy/envs/veterans-affairs.env 15 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 16 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 17 | 18 | # National Aeronautics and Space Administration 19 | source $ANALYTICS_ROOT_PATH/deploy/envs/national-aeronautics-space-administration.env 20 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 21 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 22 | 23 | # Department of Justice 24 | source $ANALYTICS_ROOT_PATH/deploy/envs/justice.env 25 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 26 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 27 | 28 | # Department of Commerce 29 | source $ANALYTICS_ROOT_PATH/deploy/envs/commerce.env 30 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 31 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 32 | 33 | # Environmental Protection Agency 34 | source $ANALYTICS_ROOT_PATH/deploy/envs/environmental-protection-agency.env 35 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 36 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 37 | 38 | # Small Business Administration 39 | source $ANALYTICS_ROOT_PATH/deploy/envs/small-business-administration.env 40 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 41 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 42 | 43 | # Department of Energy 44 | source $ANALYTICS_ROOT_PATH/deploy/envs/energy.env 45 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 46 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 47 | 48 | # Department of the Interior 49 | source $ANALYTICS_ROOT_PATH/deploy/envs/interior.env 50 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 51 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 52 | 53 | # National Archives and Records Administration 54 | source $ANALYTICS_ROOT_PATH/deploy/envs/national-archives-records-administration.env 55 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 56 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 57 | 58 | # Department of Agriculture 59 | source $ANALYTICS_ROOT_PATH/deploy/envs/agriculture.env 60 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 61 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 62 | 63 | # Department of Defense 64 | source $ANALYTICS_ROOT_PATH/deploy/envs/defense.env 65 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 66 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 67 | 68 | # Department of Health and Human Services 69 | source $ANALYTICS_ROOT_PATH/deploy/envs/health-human-services.env 70 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 71 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 72 | 73 | # Department of Housing and Urban Development 74 | source $ANALYTICS_ROOT_PATH/deploy/envs/housing-urban-development.env 75 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 76 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 77 | 78 | # Department of Homeland Security 79 | source $ANALYTICS_ROOT_PATH/deploy/envs/homeland-security.env 80 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 81 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 82 | 83 | # Department of Labor 84 | source $ANALYTICS_ROOT_PATH/deploy/envs/labor.env 85 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 86 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 87 | 88 | # Department of State 89 | source $ANALYTICS_ROOT_PATH/deploy/envs/state.env 90 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 91 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 92 | 93 | # Department of Transportation 94 | source $ANALYTICS_ROOT_PATH/deploy/envs/transportation.env 95 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 96 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 97 | 98 | # Department of the Treasury 99 | source $ANALYTICS_ROOT_PATH/deploy/envs/treasury.env 100 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 101 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 102 | 103 | # Agency for International Development 104 | source $ANALYTICS_ROOT_PATH/deploy/envs/agency-international-development.env 105 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 106 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 107 | 108 | # General Services Administration 109 | source $ANALYTICS_ROOT_PATH/deploy/envs/general-services-administration.env 110 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 111 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 112 | 113 | # National Science Foundation 114 | source $ANALYTICS_ROOT_PATH/deploy/envs/national-science-foundation.env 115 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 116 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 117 | 118 | # Nuclear Regulatory Commission 119 | source $ANALYTICS_ROOT_PATH/deploy/envs/nuclear-regulatory-commission.env 120 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 121 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 122 | 123 | # Office of Personnel Management 124 | source $ANALYTICS_ROOT_PATH/deploy/envs/office-personnel-management.env 125 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 126 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 127 | 128 | # Social Security Administration 129 | source $ANALYTICS_ROOT_PATH/deploy/envs/social-security-administration.env 130 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 131 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 132 | 133 | # Postal Service 134 | source $ANALYTICS_ROOT_PATH/deploy/envs/postal-service.env 135 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 136 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 137 | 138 | # Executive Office of the President 139 | source $ANALYTICS_ROOT_PATH/deploy/envs/executive-office-president.env 140 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --frequency=realtime --slim --verbose 141 | $ANALYTICS_ROOT_PATH/bin/analytics --publish --only=all-pages-realtime --slim --verbose --csv 142 | -------------------------------------------------------------------------------- /reports/api.json: -------------------------------------------------------------------------------- 1 | { 2 | "reports": [ 3 | { 4 | "name": "device", 5 | "frequency": "daily", 6 | "query": { 7 | "dimensions": ["ga:date" ,"ga:deviceCategory"], 8 | "metrics": ["ga:sessions"], 9 | "start-date": "3daysAgo", 10 | "end-date": "yesterday", 11 | "sort": "-ga:sessions", 12 | "filters": ["ga:sessions>100"] 13 | }, 14 | "meta": { 15 | "name": "Devices", 16 | "description": "Desktop/mobile/tablet visits" 17 | } 18 | }, 19 | { 20 | "name": "language", 21 | "frequency": "daily", 22 | "query": { 23 | "dimensions": ["ga:date" ,"ga:language"], 24 | "metrics": ["ga:sessions"], 25 | "start-date": "3daysAgo", 26 | "end-date": "yesterday", 27 | "sort": "-ga:sessions", 28 | "filters": [ 29 | "ga:sessions>100" 30 | ] 31 | }, 32 | "meta": { 33 | "name": "Browser Languages", 34 | "description": "Browser languages of visiting browsers" 35 | } 36 | }, 37 | { 38 | "name": "device-model", 39 | "frequency": "daily", 40 | "query": { 41 | "dimensions": ["ga:date" ,"ga:mobileDeviceModel"], 42 | "metrics": ["ga:sessions"], 43 | "start-date": "3daysAgo", 44 | "end-date": "yesterday", 45 | "sort": "-ga:sessions", 46 | "filters": ["ga:sessions>100"] 47 | }, 48 | "meta": { 49 | "name": "Device Models", 50 | "description": "Device models of visiting devices" 51 | } 52 | }, 53 | { 54 | "name": "os", 55 | "frequency": "daily", 56 | "query": { 57 | "dimensions": ["ga:date" ,"ga:operatingSystem"], 58 | "metrics": ["ga:sessions"], 59 | "start-date": "3daysAgo", 60 | "end-date": "yesterday", 61 | "sort": "-ga:sessions", 62 | "filters": ["ga:sessions>100"] 63 | }, 64 | "meta": { 65 | "name": "Operating Systems", 66 | "description": "Operating systems of visiting devices" 67 | } 68 | }, 69 | { 70 | "name": "windows", 71 | "frequency": "daily", 72 | "query": { 73 | "dimensions": ["ga:date" ,"ga:operatingSystemVersion"], 74 | "metrics": ["ga:sessions"], 75 | "start-date": "3daysAgo", 76 | "end-date": "yesterday", 77 | "sort": "-ga:sessions", 78 | "filters": [ 79 | "ga:operatingSystem==Windows", 80 | "ga:sessions>100" 81 | ] 82 | }, 83 | "meta": { 84 | "name": "Windows", 85 | "description": "Operating system of visiting windows devices" 86 | } 87 | }, 88 | { 89 | "name": "browser", 90 | "frequency": "daily", 91 | "query": { 92 | "dimensions": ["ga:date" ,"ga:browser"], 93 | "metrics": ["ga:sessions"], 94 | "start-date": "3daysAgo", 95 | "end-date": "yesterday", 96 | "sort": "-ga:sessions", 97 | "filters": ["ga:sessions>100"] 98 | }, 99 | "meta": { 100 | "name": "Browsers", 101 | "description": "Browsers of visiting users" 102 | } 103 | }, 104 | { 105 | "name": "ie", 106 | "frequency": "daily", 107 | "query": { 108 | "dimensions": ["ga:date","ga:browserVersion"], 109 | "metrics": ["ga:sessions"], 110 | "start-date": "3daysAgo", 111 | "end-date": "yesterday", 112 | "sort": "-ga:sessions", 113 | "filters": [ 114 | "ga:browser==Internet Explorer", 115 | "ga:sessions>100" 116 | ] 117 | }, 118 | "meta": { 119 | "name": "Internet Explorer", 120 | "description": "Visits from Internet Explorer users broken down by version" 121 | } 122 | }, 123 | { 124 | "name": "os-browser", 125 | "frequency": "daily", 126 | "query": { 127 | "dimensions": ["ga:date" ,"ga:browser", "ga:operatingSystem"], 128 | "metrics": ["ga:sessions"], 129 | "start-date": "3daysAgo", 130 | "end-date": "yesterday", 131 | "sort": "-ga:sessions", 132 | "filters": ["ga:sessions>100"] 133 | }, 134 | "meta": { 135 | "name": "OS-browser combinations", 136 | "description": "Visits broken down by browser and OS for all sites" 137 | } 138 | }, 139 | { 140 | "name": "windows-browser", 141 | "frequency": "daily", 142 | "query": { 143 | "dimensions": ["ga:date" ,"ga:browser", "ga:operatingSystemVersion"], 144 | "metrics": ["ga:sessions"], 145 | "start-date": "3daysAgo", 146 | "end-date": "yesterday", 147 | "sort": "-ga:sessions", 148 | "filters": [ 149 | "ga:sessions>100", 150 | "ga:operatingSystem==Windows" 151 | ] 152 | }, 153 | "meta": { 154 | "name": "Windows-browser combinations", 155 | "description": "Visits broken down by Windows versions and browser for all sites" 156 | } 157 | }, 158 | { 159 | "name": "windows-ie", 160 | "frequency": "daily", 161 | "query": { 162 | "dimensions": ["ga:date","ga:browserVersion", "ga:operatingSystemVersion"], 163 | "metrics": ["ga:sessions"], 164 | "start-date": "3daysAgo", 165 | "end-date": "yesterday", 166 | "sort": "-ga:sessions", 167 | "filters": [ 168 | "ga:sessions>100", 169 | "ga:browser==Internet Explorer", 170 | "ga:operatingSystem==Windows" 171 | ] 172 | }, 173 | "meta": { 174 | "name": "IE on Windows", 175 | "description": "Visits from IE on Windows broken down by IE and Windows versions" 176 | } 177 | }, 178 | { 179 | "name": "domain", 180 | "frequency": "daily", 181 | "query": { 182 | "dimensions": ["ga:date", "ga:hostname"], 183 | "metrics": ["ga:sessions"], 184 | "start-date": "3daysAgo", 185 | "end-date": "yesterday", 186 | "sort": "-ga:sessions", 187 | "filters": ["ga:sessions>100"] 188 | }, 189 | "meta": { 190 | "name": "Domains", 191 | "description": "Number of visitors for a given domain" 192 | } 193 | }, 194 | { 195 | "name": "traffic-source", 196 | "frequency": "daily", 197 | "query": { 198 | "dimensions": ["ga:date", "ga:source", "ga:hasSocialSourceReferral"], 199 | "metrics": ["ga:sessions"], 200 | "start-date": "3daysAgo", 201 | "end-date": "yesterday", 202 | "sort": "-ga:sessions", 203 | "filters": ["ga:sessions>100"] 204 | }, 205 | "meta": { 206 | "name": "Top Traffic Sources", 207 | "description": "Visitors for a given traffic source" 208 | } 209 | }, 210 | { 211 | "name": "second-level-domain", 212 | "frequency": "daily", 213 | "query": { 214 | "dimensions": ["ga:date", "ga:hostname"], 215 | "metrics": ["ga:sessions"], 216 | "sort": "-ga:sessions", 217 | "filters": [ 218 | "ga:sessions>100", 219 | "ga:hostname=~^[^\\.]+\\.[^\\.]+$" 220 | ], 221 | "start-date": "3daysAgo", 222 | "end-date": "yesterday" 223 | }, 224 | "meta": { 225 | "name": "Participating second-level domains.", 226 | "description": "Visits to participating second-level domains" 227 | } 228 | }, 229 | { 230 | "name": "site", 231 | "frequency": "daily", 232 | "query": { 233 | "dimensions": ["ga:date", "ga:hostname"], 234 | "metrics": ["ga:sessions"], 235 | "filters": ["ga:sessions>100"], 236 | "start-date": "3daysAgo", 237 | "sort": "-ga:sessions", 238 | "end-date": "yesterday" 239 | }, 240 | "meta": { 241 | "name": "Participating hostnames.", 242 | "description": "Visits to participating hostnames" 243 | } 244 | }, 245 | { 246 | "name": "download", 247 | "frequency": "daily", 248 | "query": { 249 | "dimensions": ["ga:date", "ga:pageTitle", "ga:eventLabel", "ga:pagePath"], 250 | "metrics": ["ga:totalEvents"], 251 | "filters": [ 252 | "ga:eventCategory=~ownload", 253 | "ga:pagePath!~(usps.com).*\/(?i)(zip|doc).*", 254 | "ga:totalEvents>100" 255 | ], 256 | "start-date": "3daysAgo", 257 | "sort": "-ga:totalEvents", 258 | "end-date": "yesterday" 259 | }, 260 | "meta": { 261 | "name": "Downloads", 262 | "description": "Number of download events" 263 | } 264 | } 265 | ] 266 | } 267 | -------------------------------------------------------------------------------- /test/index.test.js: -------------------------------------------------------------------------------- 1 | const expect = require("chai").expect 2 | const proxyquire = require("proxyquire") 3 | const resultFixture = require("./support/fixtures/results") 4 | 5 | describe("main", () => { 6 | describe(".run(options)", () => { 7 | const consoleLogOriginal = console.log 8 | after(() => { 9 | console.log = consoleLogOriginal 10 | }) 11 | 12 | const config = {} 13 | 14 | let Analytics 15 | let DiskPublisher 16 | let PostgresPublisher 17 | let ResultFormatter 18 | let S3Publisher 19 | let result 20 | let main 21 | 22 | beforeEach(() => { 23 | result = {} 24 | Analytics = { 25 | reports: [{ name: "a" }, { name: "b" }, { name: "c" }], 26 | query: (report) => Promise.resolve(Object.assign(result, { name: report.name })), 27 | } 28 | DiskPublisher = {} 29 | PostgresPublisher = {} 30 | ResultFormatter = { 31 | formatResult: (result) => Promise.resolve(JSON.stringify(result)) 32 | } 33 | S3Publisher = {} 34 | 35 | main = proxyquire("../index.js", { 36 | "./src/config": config, 37 | "./src/analytics": Analytics, 38 | "./src/publish/disk": DiskPublisher, 39 | "./src/publish/postgres": PostgresPublisher, 40 | "./src/process-results/result-formatter": ResultFormatter, 41 | "./src/publish/s3": S3Publisher, 42 | }) 43 | }) 44 | 45 | it("should query for every single report", done => { 46 | const queriedReportNames = [] 47 | 48 | Analytics.query = (report) => { 49 | queriedReportNames.push(report.name) 50 | return Promise.resolve(result) 51 | } 52 | 53 | main.run().then(() => { 54 | expect(queriedReportNames).to.include.members(["a", "b", "c"]) 55 | done() 56 | }).catch(done) 57 | }) 58 | 59 | it("should log formatted results", done => { 60 | ResultFormatter.formatResult = () => Promise.resolve("I'm the results!") 61 | 62 | let consoleLogCalled = false 63 | console.log = function(output) { 64 | if (output === "I'm the results!") { 65 | consoleLogCalled = true 66 | } else { 67 | consoleLogOriginal.apply(this, arguments) 68 | } 69 | } 70 | 71 | main.run().then(() => { 72 | console.log = consoleLogOriginal 73 | expect(consoleLogCalled).to.be.true 74 | done() 75 | }).catch(err => { 76 | console.log = consoleLogOriginal 77 | done(err) 78 | }) 79 | }) 80 | 81 | it("should format the results with the format set to JSON", done => { 82 | let formatResultCalled = false 83 | ResultFormatter.formatResult = (result, options) => { 84 | expect(options.format).to.equal("json") 85 | formatResultCalled = true 86 | return Promise.resolve("") 87 | } 88 | 89 | main.run().then(() => { 90 | expect(formatResultCalled).to.be.true 91 | done() 92 | }).catch(done) 93 | }) 94 | 95 | context("with --output option", () => { 96 | it("should write the results to the given path folder", done => { 97 | ResultFormatter.formatResult = () => Promise.resolve("I'm the result") 98 | 99 | const writtenReportNames = [] 100 | DiskPublisher.publish = (report, formattedResult, options) => { 101 | expect(options.format).to.equal("json") 102 | expect(options.output).to.equal("path/to/output") 103 | expect(formattedResult).to.equal("I'm the result") 104 | writtenReportNames.push(report.name) 105 | } 106 | 107 | main.run({ output: "path/to/output" }).then(() => { 108 | expect(writtenReportNames).to.include.members(["a", "b", "c"]) 109 | done() 110 | }).catch(done) 111 | }) 112 | }) 113 | 114 | context("with --publish option", () => { 115 | it("should publish the results to s3", done => { 116 | result = { data: "I'm the result" } 117 | 118 | const publishedReportNames = [] 119 | S3Publisher.publish = (report, formattedResult, options) => { 120 | expect(options.format).to.equal("json") 121 | expect(JSON.parse(formattedResult)).to.deep.equal(result) 122 | publishedReportNames.push(report.name) 123 | } 124 | 125 | main.run({ publish: true }).then(() => { 126 | expect(publishedReportNames).to.include.members(["a", "b", "c"]) 127 | done() 128 | }).catch(done) 129 | }) 130 | }) 131 | 132 | context("with --write-to-database option", () => { 133 | it("should write the results to postgres", done => { 134 | result = { data: "I am the result" } 135 | 136 | let publishCalled = false 137 | PostgresPublisher.publish = (resultToPublish) => { 138 | expect(resultToPublish).to.deep.equal(result) 139 | publishCalled = true 140 | return Promise.resolve() 141 | } 142 | 143 | main.run({ ["write-to-database"]: true }).then(() => { 144 | expect(publishCalled).to.be.true 145 | done() 146 | }).catch(done) 147 | }) 148 | 149 | it("should not write the results to postgres if the report is realtime", done => { 150 | let publishCalled = false 151 | PostgresPublisher.publish = () => { 152 | publishCalled = true 153 | return Promise.resolve() 154 | } 155 | 156 | main.run({ ["write-to-database"]: false }).then(() => { 157 | expect(publishCalled).to.be.false 158 | done() 159 | }).catch(done) 160 | }) 161 | }) 162 | 163 | context("with --only option", () => { 164 | it("should only query the given report", done => { 165 | const queriedReportNames = [] 166 | 167 | Analytics.query = (report) => { 168 | queriedReportNames.push(report.name) 169 | return Promise.resolve(result) 170 | } 171 | 172 | main.run({ only: "a" }).then(() => { 173 | expect(queriedReportNames).to.include("a") 174 | expect(queriedReportNames).not.to.include.members(["b", "c"]) 175 | done() 176 | }).catch(done) 177 | }) 178 | }) 179 | 180 | context("with --slim option", () => { 181 | it("should format the results with the slim option for slim reports", done => { 182 | Analytics.reports = [ 183 | { name: "a", slim: false }, 184 | { name: "b", slim: true }, 185 | { name: "c", slim: false }, 186 | ] 187 | 188 | const formattedSlimReportNames = [] 189 | const formattedRegularReportNames = [] 190 | ResultFormatter.formatResult = (result, options) => { 191 | if (options.slim === true) { 192 | formattedSlimReportNames.push(result.name) 193 | } else { 194 | formattedRegularReportNames.push(result.name) 195 | } 196 | return Promise.resolve("") 197 | } 198 | 199 | main.run({ slim: true }).then(() => { 200 | expect(formattedSlimReportNames).to.include.members(["b"]) 201 | expect(formattedRegularReportNames).to.include.members(["a", "c"]) 202 | done() 203 | }).catch(done) 204 | }) 205 | }) 206 | 207 | context("with --csv option", () => { 208 | it("should format the reports with the format set to csv", done => { 209 | const formattedReportNames = [] 210 | ResultFormatter.formatResult = (result, options) => { 211 | expect(options.format).to.equal("csv") 212 | formattedReportNames.push(result.name) 213 | return Promise.resolve("") 214 | } 215 | 216 | main.run({ csv: true }).then(() => { 217 | expect(formattedReportNames).to.include.members(["a", "b", "c"]) 218 | done() 219 | }).catch(done) 220 | }) 221 | 222 | it("should publish the reports with the format set to csv", done => { 223 | result = { data: "I'm the result" } 224 | 225 | const publishedReportNames = [] 226 | S3Publisher.publish = (report, formattedResult, options) => { 227 | expect(options.format).to.equal("csv") 228 | publishedReportNames.push(report.name) 229 | } 230 | 231 | main.run({ publish: true, csv: true }).then(() => { 232 | expect(publishedReportNames).to.include.members(["a", "b", "c"]) 233 | done() 234 | }).catch(done) 235 | }) 236 | }) 237 | 238 | context("with --frequency option", () => { 239 | it("should only query reports with the given frequency", done => { 240 | Analytics.reports = [ 241 | { name: "a", frequency: "daily" }, 242 | { name: "b", frequency: "hourly" }, 243 | { name: "c", frequency: "daily" }, 244 | ] 245 | 246 | const queriedReportNames = [] 247 | 248 | Analytics.query = (report) => { 249 | queriedReportNames.push(report.name) 250 | return Promise.resolve(result) 251 | } 252 | 253 | main.run({ frequency: "daily" }).then(() => { 254 | expect(queriedReportNames).to.include.members(["a", "c"]) 255 | expect(queriedReportNames).not.to.include.members(["b"]) 256 | done() 257 | }).catch(done) 258 | }) 259 | }) 260 | }) 261 | }) 262 | -------------------------------------------------------------------------------- /test/process-results/result-totals-calculator.test.js: -------------------------------------------------------------------------------- 1 | const expect = require("chai").expect 2 | const proxyquire = require("proxyquire") 3 | const reportFixture = require("../support/fixtures/report") 4 | const dataFixture = require("../support/fixtures/data") 5 | const ResultTotalsCalculator = require("../../src/process-results/result-totals-calculator") 6 | 7 | proxyquire.noCallThru() 8 | 9 | const GoogleAnalyticsDataProcessor = proxyquire("../../src/process-results/ga-data-processor", { 10 | "../config": { account: { hostname: "" } }, 11 | }) 12 | 13 | describe("ResultTotalsCalculator", () => { 14 | describe("calculateTotals(result)", () => { 15 | let report 16 | let data 17 | 18 | beforeEach(() => { 19 | report = Object.assign({}, reportFixture) 20 | data = Object.assign({}, dataFixture) 21 | }) 22 | 23 | it("should compute totals for users", () => { 24 | data.columnHeaders = [{ name: "ga:users" }] 25 | data.rows = [["10"], ["15"], ["20"]] 26 | 27 | const result = GoogleAnalyticsDataProcessor.processData(report, data) 28 | 29 | const totals = ResultTotalsCalculator.calculateTotals(result) 30 | expect(totals.users).to.equal(10 + 15 + 20) 31 | }) 32 | 33 | it("should compute totals for visits", () => { 34 | data.columnHeaders = [{ name: "ga:sessions" }] 35 | data.rows = [["10"], ["15"], ["20"]] 36 | 37 | const result = GoogleAnalyticsDataProcessor.processData(report, data) 38 | 39 | const totals = ResultTotalsCalculator.calculateTotals(result) 40 | expect(totals.visits).to.equal(10 + 15 + 20) 41 | }) 42 | 43 | it("should compute totals for device_models", () => { 44 | report.name = "device_model" 45 | data.columnHeaders = [ 46 | { name: "ga:date" }, 47 | { name: "ga:mobileDeviceModel" }, 48 | { name: "ga:sessions" }, 49 | ] 50 | data.rows = [ 51 | ["20170130", "iPhone", "100"], 52 | ["20170130", "Android", "200"], 53 | ["20170131", "iPhone", "300"], 54 | ["20170131", "Android", "400"], 55 | ] 56 | 57 | const result = GoogleAnalyticsDataProcessor.processData(report, data) 58 | 59 | const totals = ResultTotalsCalculator.calculateTotals(result) 60 | expect(totals.device_models.iPhone).to.equal(100 + 300) 61 | expect(totals.device_models.Android).to.equal(200 + 400) 62 | }) 63 | 64 | it("should compute totals for languages", () => { 65 | report.name = "language" 66 | data.columnHeaders = [ 67 | { name: "ga:date" }, 68 | { name: "ga:language" }, 69 | { name: "ga:sessions" }, 70 | ] 71 | data.rows = [ 72 | ["20170130", "en", "100"], 73 | ["20170130", "es", "200"], 74 | ["20170131", "en", "300"], 75 | ["20170131", "es", "400"], 76 | ] 77 | 78 | const result = GoogleAnalyticsDataProcessor.processData(report, data) 79 | 80 | const totals = ResultTotalsCalculator.calculateTotals(result) 81 | expect(totals.languages.en).to.equal(100 + 300) 82 | expect(totals.languages.es).to.equal(200 + 400) 83 | }) 84 | 85 | it("should compute totals for devices", () => { 86 | report.name = "devices" 87 | data.columnHeaders = [ 88 | { name: "ga:date" }, 89 | { name: "ga:deviceCategory" }, 90 | { name: "ga:sessions" }, 91 | ] 92 | data.rows = [ 93 | ["20170130", "mobile", "100"], 94 | ["20170130", "tablet", "200"], 95 | ["20170130", "desktop", "300"], 96 | ["20170131", "mobile", "400"], 97 | ["20170131", "tablet", "500"], 98 | ["20170131", "desktop", "600"], 99 | ] 100 | 101 | const result = GoogleAnalyticsDataProcessor.processData(report, data) 102 | 103 | const totals = ResultTotalsCalculator.calculateTotals(result) 104 | expect(totals.devices.mobile).to.equal(100 + 400) 105 | expect(totals.devices.tablet).to.equal(200 + 500) 106 | expect(totals.devices.desktop).to.equal(300 + 600) 107 | }) 108 | 109 | it("should compute totals for screen-sizes", () => { 110 | report.name = "screen-size" 111 | data.columnHeaders = [ 112 | { name: "ga:date" }, 113 | { name: "ga:screenResolution" }, 114 | { name: "ga:sessions" }, 115 | ] 116 | data.rows = [ 117 | ["20170130", "100x100", "100"], 118 | ["20170130", "200x200", "200"], 119 | ["20170131", "100x100", "300"], 120 | ["20170131", "200x200", "400"], 121 | ] 122 | 123 | const result = GoogleAnalyticsDataProcessor.processData(report, data) 124 | 125 | const totals = ResultTotalsCalculator.calculateTotals(result) 126 | expect(totals.screen_resolution["100x100"]).to.equal(100 + 300) 127 | expect(totals.screen_resolution["200x200"]).to.equal(200 + 400) 128 | }) 129 | 130 | it("should compute totals for os", () => { 131 | report.name = "os" 132 | data.columnHeaders = [ 133 | { name: "ga:date" }, 134 | { name: "ga:operatingSystem" }, 135 | { name: "ga:sessions" }, 136 | ] 137 | data.rows = [ 138 | ["20170130", "Nintendo Wii", "100"], 139 | ["20170130", "Xbox", "200"], 140 | ["20170131", "Nintendo Wii", "300"], 141 | ["20170131", "Xbox", "400"], 142 | ] 143 | 144 | const result = GoogleAnalyticsDataProcessor.processData(report, data) 145 | 146 | const totals = ResultTotalsCalculator.calculateTotals(result) 147 | expect(totals.os["Nintendo Wii"]).to.equal(100 + 300) 148 | expect(totals.os["Xbox"]).to.equal(200 + 400) 149 | }) 150 | 151 | it("should compute totals for windows", () => { 152 | report.name = "windows" 153 | data.columnHeaders = [ 154 | { name: "ga:date" }, 155 | { name: "ga:operatingSystemVersion" }, 156 | { name: "ga:sessions" }, 157 | ] 158 | data.rows = [ 159 | ["20170130", "Server", "100"], 160 | ["20170130", "Vista", "200"], 161 | ["20170131", "Server", "300"], 162 | ["20170131", "Vista", "400"], 163 | ] 164 | 165 | const result = GoogleAnalyticsDataProcessor.processData(report, data) 166 | 167 | const totals = ResultTotalsCalculator.calculateTotals(result) 168 | expect(totals.os_version.Server).to.equal(100 + 300) 169 | expect(totals.os_version.Vista).to.equal(200 + 400) 170 | }) 171 | 172 | it("should compute totals for browsers", () => { 173 | report.name = "browsers" 174 | data.columnHeaders = [ 175 | { name: "ga:date" }, 176 | { name: "ga:browser" }, 177 | { name: "ga:sessions" }, 178 | ] 179 | data.rows = [ 180 | ["20170130", "Chrome", "100"], 181 | ["20170130", "Safari", "200"], 182 | ["20170131", "Chrome", "300"], 183 | ["20170131", "Safari", "400"], 184 | ] 185 | 186 | const result = GoogleAnalyticsDataProcessor.processData(report, data) 187 | 188 | const totals = ResultTotalsCalculator.calculateTotals(result) 189 | expect(totals.browser.Chrome).to.equal(100 + 300) 190 | expect(totals.browser.Safari).to.equal(200 + 400) 191 | }) 192 | 193 | it("should compute totals for ie", () => { 194 | report.name = "ie" 195 | data.columnHeaders = [ 196 | { name: "ga:date" }, 197 | { name: "ga:browserVersion" }, 198 | { name: "ga:sessions" }, 199 | ] 200 | data.rows = [ 201 | ["20170130", "10.0", "100"], 202 | ["20170130", "11.0", "200"], 203 | ["20170131", "10.0", "300"], 204 | ["20170131", "11.0", "400"], 205 | ] 206 | 207 | const result = GoogleAnalyticsDataProcessor.processData(report, data) 208 | 209 | const totals = ResultTotalsCalculator.calculateTotals(result) 210 | expect(totals.ie_version["10.0"]).to.equal(100 + 300) 211 | expect(totals.ie_version["11.0"]).to.equal(200 + 400) 212 | }) 213 | 214 | it("should compute totals for os-browsers by operating system and browser", () => { 215 | report.name = "os-browsers" 216 | data.columnHeaders = [ 217 | { name: "ga:date" }, 218 | { name: "ga:operatingSystem" }, 219 | { name: "ga:browser" }, 220 | { name: "ga:sessions" }, 221 | ] 222 | data.rows = [ 223 | ["20170130", "Windows", "Chrome", "100"], 224 | ["20170130", "Windows", "Firefox", "200"], 225 | ["20170130", "Linux", "Chrome", "300"], 226 | ["20170130", "Linux", "Firefox", "400"], 227 | ["20170130", "Windows", "Chrome", "500"], 228 | ["20170130", "Windows", "Firefox", "600"], 229 | ["20170130", "Linux", "Chrome", "700"], 230 | ["20170130", "Linux", "Firefox", "800"], 231 | ] 232 | 233 | const result = GoogleAnalyticsDataProcessor.processData(report, data) 234 | 235 | const totals = ResultTotalsCalculator.calculateTotals(result) 236 | 237 | expect(totals.by_os.Windows.Chrome).to.equal(100 + 500) 238 | expect(totals.by_os.Windows.Firefox).to.equal(200 + 600) 239 | 240 | expect(totals.by_browsers.Chrome.Windows).to.equal(100 + 500) 241 | expect(totals.by_browsers.Chrome.Linux).to.equal(300 + 700) 242 | }) 243 | 244 | it("should compute totals for windows-ie by Windows version and IE version", () => { 245 | report.name = "windows-ie" 246 | data.columnHeaders = [ 247 | { name: "ga:date" }, 248 | { name: "ga:operatingSystemVersion" }, 249 | { name: "ga:browserVersion" }, 250 | { name: "ga:sessions" }, 251 | ] 252 | data.rows = [ 253 | ["20170130", "XP", "10", "100"], 254 | ["20170130", "XP", "7", "200"], 255 | ["20170130", "Vista", "10", "300"], 256 | ["20170130", "Vista", "7", "400"], 257 | ["20170130", "XP", "10", "500"], 258 | ["20170130", "XP", "7", "600"], 259 | ["20170130", "Vista", "10", "700"], 260 | ["20170130", "Vista", "7", "800"], 261 | ] 262 | 263 | const result = GoogleAnalyticsDataProcessor.processData(report, data) 264 | 265 | const totals = ResultTotalsCalculator.calculateTotals(result) 266 | 267 | expect(totals.by_windows.XP["10"]).to.equal(100 + 500) 268 | expect(totals.by_windows.XP["7"]).to.equal(200 + 600) 269 | 270 | expect(totals.by_ie["10"].XP).to.equal(100 + 500) 271 | expect(totals.by_ie["10"].Vista).to.equal(300 + 700) 272 | }) 273 | 274 | it("should compute totals for windows-browsers by windows version and browser version", () => { 275 | report.name = "windows-browsers" 276 | data.columnHeaders = [ 277 | { name: "ga:date" }, 278 | { name: "ga:operatingSystemVersion" }, 279 | { name: "ga:browser" }, 280 | { name: "ga:sessions" }, 281 | ] 282 | data.rows = [ 283 | ["20170130", "XP", "Chrome", "100"], 284 | ["20170130", "XP", "Firefox", "200"], 285 | ["20170130", "Vista", "Chrome", "300"], 286 | ["20170130", "Vista", "Firefox", "400"], 287 | ["20170130", "XP", "Chrome", "500"], 288 | ["20170130", "XP", "Firefox", "600"], 289 | ["20170130", "Vista", "Chrome", "700"], 290 | ["20170130", "Vista", "Firefox", "800"], 291 | ] 292 | 293 | const result = GoogleAnalyticsDataProcessor.processData(report, data) 294 | 295 | const totals = ResultTotalsCalculator.calculateTotals(result) 296 | 297 | expect(totals.by_windows.XP.Chrome).to.equal(100 + 500) 298 | expect(totals.by_windows.XP.Firefox).to.equal(200 + 600) 299 | 300 | expect(totals.by_browsers.Chrome.XP).to.equal(100 + 500) 301 | expect(totals.by_browsers.Chrome.Vista).to.equal(300 + 700) 302 | }) 303 | }) 304 | }) 305 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Code Climate](https://codeclimate.com/github/18F/analytics-reporter/badges/gpa.svg)](https://codeclimate.com/github/18F/analytics-reporter) [![CircleCI](https://circleci.com/gh/18F/analytics.usa.gov.svg?style=shield)](https://circleci.com/gh/18F/analytics.usa.gov) [![Dependency Status](https://gemnasium.com/badges/github.com/18F/analytics-reporter.svg)](https://gemnasium.com/github.com/18F/analytics-reporter) 2 | 3 | ## Analytics Reporter 4 | 5 | A lightweight system for publishing analytics data from Google Analytics profiles. Uses the [Google Analytics Core Reporting API v3](https://developers.google.com/analytics/devguides/reporting/core/v3/) and the [Google Analytics Real Time API v3](https://developers.google.com/analytics/devguides/reporting/realtime/v3/). 6 | 7 | This is used in combination with [18F/analytics.usa.gov](https://github.com/18F/analytics.usa.gov) to power the government analytics hub, [analytics.usa.gov](https://analytics.usa.gov). 8 | 9 | Available reports are named and described in [`reports.json`](reports/reports.json). For now, they're hardcoded into the repository. 10 | 11 | 12 | ### Installation 13 | 14 | ### Docker 15 | 16 | * To build the docker image on your computer, run: 17 | 18 | ````bash 19 | export NODE_ENV=development # just needed when developing against the image 20 | export NODE_ENV=production # to build an image for production 21 | docker build --build-arg NODE_ENV=${NODE_ENV} -t analytics-reporter . 22 | ```` 23 | 24 | Then you can create an alias in order to have the analytics command available: 25 | 26 | ```bash 27 | alias analytics="docker run -t -v ${HOME}:${HOME} -e ANALYTICS_REPORT_EMAIL -e ANALYTICS_REPORT_IDS -e ANALYTICS_KEY analytics-reporter" 28 | ``` 29 | 30 | To make this command working as expected you should export the env vars as follows: 31 | 32 | ```bash 33 | export ANALYTICS_REPORT_EMAIL= "your-report-email" 34 | export ANALYTICS_REPORT_IDS="your-report-ids" 35 | export ANALYTICS_KEY="your-key" 36 | ``` 37 | 38 | ### NPM 39 | 40 | * To run the utility on your computer, install it through npm: 41 | 42 | ```bash 43 | npm install -g analytics-reporter 44 | ``` 45 | 46 | If you're developing locally inside the repo, `npm install` is sufficient. 47 | 48 | ### Setup 49 | 50 | * Create an API service account in the [Google developer dashboard](https://console.developers.google.com/apis/). 51 | 52 | * Visit the "APIs" section of the Google Developer Dashboard for your project, and enable it for the "Analytics API". 53 | 54 | * Go to the "Credentials" section and generate "service account" credentials using a new service account. 55 | 56 | * Download the **JSON** private key file it gives you. 57 | 58 | * Grab the generated client email address (ends with `gserviceaccount.com`) from the contents of the .json file. 59 | 60 | * Grant that email address `Read, Analyze & Collaborate` permissions on the Google Analytics profile(s) whose data you wish to publish. 61 | 62 | * Set environment variables for your app's generated email address, and for the profile you authorized it to: 63 | 64 | ```bash 65 | export ANALYTICS_REPORT_EMAIL="YYYYYYY@developer.gserviceaccount.com" 66 | export ANALYTICS_REPORT_IDS="ga:XXXXXX" 67 | ``` 68 | 69 | You may wish to manage these using [`autoenv`](https://github.com/kennethreitz/autoenv). If you do, there is an `example.env` file you can copy to `.env` to get started. 70 | 71 | To find your Google Analytics view ID: 72 | 73 | 1. Sign in to your Analytics account. 74 | 1. Select the Admin tab. 75 | 1. Select an account from the dropdown in the ACCOUNT column. 76 | 1. Select a property from the dropdown in the PROPERTY column. 77 | 1. Select a view from the dropdown in the VIEW column. 78 | 1. Click "View Settings" 79 | 1. Copy the view ID. You'll need to enter it with `ga:` as a prefix. 80 | 81 | * You can specify your private key through environment variables either as a file path, or the contents of the key (helpful for Heroku and Heroku-like systems). 82 | 83 | To specify a file path (useful in development or Linux server environments): 84 | 85 | ``` 86 | export ANALYTICS_KEY_PATH="/path/to/secret_key.json" 87 | ``` 88 | 89 | Alternatively, to specify the key directly (useful in a PaaS environment), paste in the contents of the JSON file's `private_key` field **directly and exactly**, in quotes, and **rendering actual line breaks** (not `\n`'s) (below example has been sanitized): 90 | 91 | ``` 92 | export ANALYTICS_KEY="-----BEGIN PRIVATE KEY----- 93 | [contents of key] 94 | -----END PRIVATE KEY----- 95 | " 96 | ``` 97 | 98 | If you have multiple accounts for a profile, you can set the `ANALYTICS_CREDENTIALS` variable with a JSON encoded array of those credentials and they'll be used to authorize API requests in a round-robin style. 99 | 100 | ``` 101 | export ANALYTICS_CREDENTIALS='[ 102 | { 103 | "key": "-----BEGIN PRIVATE KEY-----\n[contents of key]\n-----END PRIVATE KEY-----", 104 | "email": "email_1@example.com" 105 | }, 106 | { 107 | "key": "-----BEGIN PRIVATE KEY-----\n[contents of key]\n-----END PRIVATE KEY-----", 108 | "email": "email_2@example.com" 109 | } 110 | ]' 111 | ``` 112 | 113 | * Make sure your computer or server is syncing its time with the world over NTP. Your computer's time will need to match those on Google's servers for the authentication to work. 114 | 115 | * Test your configuration by printing a report to STDOUT: 116 | 117 | ```bash 118 | ./bin/analytics --only users 119 | ``` 120 | 121 | If you see a nicely formatted JSON file, you are all set. 122 | 123 | * (Optional) Authorize yourself for S3 publishing. 124 | 125 | If you plan to use this project's lightweight S3 publishing system, you'll need to add 6 more environment variables: 126 | 127 | ``` 128 | export AWS_REGION=us-east-1 129 | export AWS_ACCESS_KEY_ID=[your-key] 130 | export AWS_SECRET_ACCESS_KEY=[your-secret-key] 131 | 132 | export AWS_BUCKET=[your-bucket] 133 | export AWS_BUCKET_PATH=[your-path] 134 | export AWS_CACHE_TIME=0 135 | ``` 136 | 137 | There are cases where you want to use a custom object storage server compatible with Amazon S3 APIs, like [minio](https://github.com/minio/minio), in that specific case you should set an extra env variable: 138 | 139 | ``` 140 | export AWS_S3_ENDPOINT=http://your-storage-server:port 141 | ``` 142 | 143 | 144 | ### Other configuration 145 | 146 | If you use a **single domain** for all of your analytics data, then your profile is likely set to return relative paths (e.g. `/faq`) and not absolute paths when accessing real-time reports. 147 | 148 | You can set a default domain, to be returned as data in all real-time data point: 149 | 150 | ``` 151 | export ANALYTICS_HOSTNAME=https://konklone.com 152 | ``` 153 | 154 | This will produce points similar to the following: 155 | 156 | ```json 157 | { 158 | "page": "/post/why-google-is-hurrying-the-web-to-kill-sha-1", 159 | "page_title": "Why Google is Hurrying the Web to Kill SHA-1", 160 | "active_visitors": "1", 161 | "domain": "https://konklone.com" 162 | } 163 | ``` 164 | 165 | ### Use 166 | 167 | Reports are created and published using the `analytics` command. 168 | 169 | ```bash 170 | analytics 171 | ``` 172 | 173 | This will run every report, in sequence, and print out the resulting JSON to STDOUT. There will be two newlines between each report. 174 | 175 | A report might look something like this: 176 | 177 | ```javascript 178 | { 179 | "name": "devices", 180 | "query": { 181 | "dimensions": [ 182 | "ga:date", 183 | "ga:deviceCategory" 184 | ], 185 | "metrics": [ 186 | "ga:sessions" 187 | ], 188 | "start-date": "90daysAgo", 189 | "end-date": "yesterday", 190 | "sort": "ga:date" 191 | }, 192 | "meta": { 193 | "name": "Devices", 194 | "description": "Weekly desktop/mobile/tablet visits by day for all sites." 195 | }, 196 | "data": [ 197 | { 198 | "date": "2014-10-14", 199 | "device": "desktop", 200 | "visits": "11495462" 201 | }, 202 | { 203 | "date": "2014-10-14", 204 | "device": "mobile", 205 | "visits": "2499586" 206 | }, 207 | { 208 | "date": "2014-10-14", 209 | "device": "tablet", 210 | "visits": "976396" 211 | }, 212 | // ... 213 | ], 214 | "totals": { 215 | "devices": { 216 | "mobile": 213920363, 217 | "desktop": 755511646, 218 | "tablet": 81874189 219 | }, 220 | "start_date": "2014-10-14", 221 | "end_date": "2015-01-11" 222 | } 223 | } 224 | ``` 225 | 226 | #### Options 227 | 228 | * `--output` - Output to a directory. 229 | 230 | ```bash 231 | analytics --output /path/to/data 232 | ``` 233 | 234 | *Note that when using the docker image you have to use the absolute path, for example "/home/youruser/path/to/data"* 235 | 236 | * `--publish` - Publish to an S3 bucket. Requires AWS environment variables set as described above. 237 | 238 | ```bash 239 | analytics --publish 240 | ``` 241 | 242 | * `--write-to-database` - write data to a database. Requires a postgres configuration to be set in environment variables as described below. 243 | 244 | * `--only` - only run one or more specific reports. Multiple reports are comma separated. 245 | 246 | ```bash 247 | analytics --only devices 248 | analytics --only devices,today 249 | ``` 250 | 251 | * `--slim` -Where supported, use totals only (omit the `data` array). Only applies to JSON, and reports where `"slim": true`. 252 | 253 | ```bash 254 | analytics --only devices --slim 255 | ``` 256 | 257 | * `--csv` - Gives you CSV instead of JSON. 258 | 259 | ```bash 260 | analytics --csv 261 | ``` 262 | 263 | * `--frequency` - Limit to reports with this 'frequency' value. 264 | 265 | ```bash 266 | analytics --frequency=realtime 267 | ``` 268 | 269 | * `--debug` - Print debug details on STDOUT. 270 | 271 | ```bash 272 | analytics --publish --debug 273 | ``` 274 | 275 | ### Saving data to postgres 276 | 277 | The analytics reporter can write data is pulls from Google Analytics to a 278 | Postgres database. The postgres configuration can be set using environment 279 | variables: 280 | 281 | ```bash 282 | export POSTGRES_HOST = "my.db.host.com" 283 | export POSTGRES_USER = "postgres" 284 | export POSTGRES_PASSWORD = "123abc" 285 | export POSTGRES_DATABASE = "analytics" 286 | ``` 287 | 288 | The database expects a particular schema which will be described in the API 289 | server that consumes this data. 290 | 291 | To write reports to a database, use the `--write-to-database` option when 292 | starting the reporter. 293 | 294 | ### Deploying to GovCloud 295 | 296 | The analytics reporter runs on :cloud:.gov. Please refer to the `manifest.yml` 297 | file at the root of the repository for application information. 298 | 299 | Ensure you're targeting the proper `org` and `space`. 300 | 301 | ```shell 302 | cf target 303 | ``` 304 | 305 | Deploy the application with the following command. 306 | 307 | ```shell 308 | cf push -f manifest.yml 309 | ``` 310 | 311 | Set the environmental variables based on local `.env` file. 312 | 313 | ```shell 314 | cf set-env analytics-reporter AWS_ACCESS_KEY_ID 123abc 315 | cf set-env analytics-reporter AWS_SECRET_ACCESS_KEY 456def 316 | # ... 317 | ``` 318 | 319 | Restage the application to use the environment variables. 320 | 321 | ```shell 322 | cf restage analytics-reporter 323 | ``` 324 | 325 | ### Developing with Docker 326 | 327 | This repo contains a [Docker Compose](https://docs.docker.com/compose/) 328 | configuration. The reporter is configured to run in the container as if it were 329 | running in GovCloud. This is helpful for seeing how the reporter will behave 330 | when deployed without pushing it to cloud.gov. 331 | 332 | To start the reporter, first run the `docker-update` script to install the 333 | necessary dependencies: 334 | 335 | ```shell 336 | ./bin/docker-update 337 | ``` 338 | 339 | Note that this script will need to be run again when new dependencies are added 340 | to update the Docker volumes where the dependencies are stored. 341 | 342 | After the dependencies are installed, the reporter can be started using Docker 343 | Compose: 344 | 345 | ```shell 346 | docker-compose up 347 | ``` 348 | 349 | ### Public domain 350 | 351 | This project is in the worldwide [public domain](LICENSE.md). As stated in [CONTRIBUTING](CONTRIBUTING.md): 352 | 353 | > This project is in the public domain within the United States, and copyright and related rights in the work worldwide are waived through the [CC0 1.0 Universal public domain dedication](https://creativecommons.org/publicdomain/zero/1.0/). 354 | > 355 | > All contributions to this project will be released under the CC0 dedication. By submitting a pull request, you are agreeing to comply with this waiver of copyright interest. 356 | -------------------------------------------------------------------------------- /reports/reports.json: -------------------------------------------------------------------------------- 1 | { 2 | "reports": [ 3 | { 4 | "name": "realtime", 5 | "frequency": "realtime", 6 | "realtime": true, 7 | "query": { 8 | "metrics": ["rt:activeUsers"] 9 | }, 10 | "meta": { 11 | "name": "Active Users Right Now", 12 | "description": "Number of users currently visiting all sites." 13 | } 14 | }, 15 | { 16 | "name": "today", 17 | "frequency": "realtime", 18 | "query": { 19 | "dimensions": ["ga:date", "ga:hour"], 20 | "metrics": ["ga:sessions"], 21 | "start-date": "today", 22 | "end-date": "today" 23 | }, 24 | "meta": { 25 | "name": "Today", 26 | "description": "Today's visits for all sites." 27 | } 28 | }, 29 | { 30 | "name": "last-48-hours", 31 | "frequency": "realtime", 32 | "query": { 33 | "dimensions": ["ga:date", "ga:hour"], 34 | "metrics": ["ga:sessions"], 35 | "start-date": "yesterday", 36 | "end-date": "today" 37 | }, 38 | "meta": { 39 | "name": "Today", 40 | "description": "Today's visits for all sites." 41 | } 42 | }, 43 | { 44 | "name": "users", 45 | "frequency": "daily", 46 | "slim": true, 47 | "query": { 48 | "dimensions": ["ga:date"], 49 | "metrics": ["ga:sessions"], 50 | "start-date": "90daysAgo", 51 | "end-date": "yesterday", 52 | "sort": "ga:date" 53 | }, 54 | "meta": { 55 | "name": "Visitors", 56 | "description": "90 days of visits for all sites." 57 | } 58 | }, 59 | { 60 | "name": "devices", 61 | "frequency": "daily", 62 | "slim": true, 63 | "query": { 64 | "dimensions": ["ga:date" ,"ga:deviceCategory"], 65 | "metrics": ["ga:sessions"], 66 | "start-date": "90daysAgo", 67 | "end-date": "yesterday", 68 | "sort": "ga:date" 69 | }, 70 | "meta": { 71 | "name": "Devices", 72 | "description": "90 days of desktop/mobile/tablet visits for all sites." 73 | } 74 | }, 75 | { 76 | "name": "os", 77 | "frequency": "daily", 78 | "slim": true, 79 | "query": { 80 | "dimensions": ["ga:date" ,"ga:operatingSystem"], 81 | "metrics": ["ga:sessions"], 82 | "start-date": "90daysAgo", 83 | "end-date": "yesterday", 84 | "sort": "ga:date" 85 | }, 86 | "meta": { 87 | "names": "Operating Systems", 88 | "description": "90 days of visits, broken down by operating system and date, for all sites." 89 | } 90 | }, 91 | { 92 | "name": "screen-size", 93 | "frequency": "daily", 94 | "slim": true, 95 | "query": { 96 | "dimensions": ["ga:date" ,"ga:screenResolution"], 97 | "metrics": ["ga:sessions"], 98 | "start-date": "90daysAgo", 99 | "end-date": "yesterday", 100 | "sort": "ga:date" 101 | }, 102 | "meta": { 103 | "name": "Screen Resolution", 104 | "description": "90 days of Screen Resolution visits for all sites." 105 | } 106 | }, 107 | { 108 | "name": "language", 109 | "frequency": "daily", 110 | "slim": true, 111 | "query": { 112 | "dimensions": ["ga:date" ,"ga:language"], 113 | "metrics": ["ga:sessions"], 114 | "start-date": "90daysAgo", 115 | "end-date": "yesterday", 116 | "sort": "ga:date" 117 | }, 118 | "meta": { 119 | "name": "Browser Language", 120 | "description": "90 days of visits by browser language for all sites." 121 | } 122 | }, 123 | { 124 | "name": "device_model", 125 | "frequency": "daily", 126 | "slim": true, 127 | "query": { 128 | "dimensions": ["ga:date" ,"ga:mobileDeviceModel"], 129 | "metrics": ["ga:sessions"], 130 | "start-date": "90daysAgo", 131 | "end-date": "yesterday", 132 | "sort": "ga:date" 133 | }, 134 | "meta": { 135 | "name": "Device Model", 136 | "description": "90 days of visits by Device Model for all sites." 137 | } 138 | }, 139 | { 140 | "name": "windows", 141 | "frequency": "daily", 142 | "slim": true, 143 | "query": { 144 | "dimensions": ["ga:date" ,"ga:operatingSystemVersion"], 145 | "metrics": ["ga:sessions"], 146 | "start-date": "90daysAgo", 147 | "end-date": "yesterday", 148 | "filters": ["ga:operatingSystem==Windows"], 149 | "sort": "ga:date" 150 | }, 151 | "meta": { 152 | "names": "Windows", 153 | "description": "90 days of visits from Windows users, broken down by operating system version and date, for all sites." 154 | } 155 | }, 156 | { 157 | "name": "browsers", 158 | "frequency": "daily", 159 | "slim": true, 160 | "query": { 161 | "dimensions": ["ga:date" ,"ga:browser"], 162 | "metrics": ["ga:sessions"], 163 | "start-date": "90daysAgo", 164 | "end-date": "yesterday", 165 | "sort": "ga:date,-ga:sessions" 166 | }, 167 | "meta": { 168 | "name": "Browsers", 169 | "description": "90 days of visits broken down by browser for all sites." 170 | } 171 | }, 172 | { 173 | "name": "ie", 174 | "frequency": "daily", 175 | "slim": true, 176 | "query": { 177 | "dimensions": ["ga:date","ga:browserVersion"], 178 | "metrics": ["ga:sessions"], 179 | "start-date": "90daysAgo", 180 | "end-date": "yesterday", 181 | "sort": "ga:date,-ga:sessions", 182 | "filters": ["ga:browser==Internet Explorer"] 183 | }, 184 | "meta": { 185 | "name": "Internet Explorer", 186 | "description": "90 days of visits from Internet Explorer users broken down by version for all sites." 187 | } 188 | }, 189 | { 190 | "name": "os-browsers", 191 | "frequency": "daily", 192 | "slim": true, 193 | "query": { 194 | "dimensions": ["ga:date" ,"ga:browser", "ga:operatingSystem"], 195 | "metrics": ["ga:sessions"], 196 | "start-date": "90daysAgo", 197 | "end-date": "yesterday", 198 | "sort": "ga:date,-ga:sessions" 199 | }, 200 | "meta": { 201 | "name": "OS-browser combinations", 202 | "description": "90 days of visits broken down by browser and OS for all sites." 203 | } 204 | }, 205 | { 206 | "name": "windows-browsers", 207 | "frequency": "daily", 208 | "slim": true, 209 | "query": { 210 | "dimensions": ["ga:date" ,"ga:browser", "ga:operatingSystemVersion"], 211 | "metrics": ["ga:sessions"], 212 | "start-date": "90daysAgo", 213 | "end-date": "yesterday", 214 | "sort": "ga:date,-ga:sessions", 215 | "filters": [ 216 | "ga:operatingSystem==Windows" 217 | ] 218 | }, 219 | "meta": { 220 | "name": "Windows-browser combinations", 221 | "description": "90 days of visits broken down by Windows versions and browser for all sites." 222 | } 223 | }, 224 | { 225 | "name": "windows-ie", 226 | "frequency": "daily", 227 | "slim": true, 228 | "query": { 229 | "dimensions": ["ga:date","ga:browserVersion", "ga:operatingSystemVersion"], 230 | "metrics": ["ga:sessions"], 231 | "start-date": "90daysAgo", 232 | "end-date": "yesterday", 233 | "sort": "ga:date,-ga:sessions", 234 | "filters": [ 235 | "ga:browser==Internet Explorer", 236 | "ga:operatingSystem==Windows" 237 | ] 238 | }, 239 | "meta": { 240 | "name": "IE on Windows", 241 | "description": "90 days of visits from IE on Windows broken down by IE and Windows versions for all sites." 242 | } 243 | }, 244 | { 245 | "name": "top-pages-realtime", 246 | "frequency": "realtime", 247 | "realtime": true, 248 | "query": { 249 | "dimensions": ["rt:pagePath", "rt:pageTitle"], 250 | "metrics": ["rt:activeUsers"], 251 | "sort": "-rt:activeUsers", 252 | "max-results": "20" 253 | }, 254 | "meta": { 255 | "name": "Top Pages (Live)", 256 | "description": "The top 20 pages, measured by active onsite users, for all sites." 257 | } 258 | }, 259 | { 260 | "name": "top-pages-7-days", 261 | "frequency": "daily", 262 | "query": { 263 | "dimensions": ["ga:hostname", "ga:pagePath", "ga:pageTitle"], 264 | "metrics": ["ga:pageviews"], 265 | "start-date": "7daysAgo", 266 | "end-date": "yesterday", 267 | "sort": "-ga:pageviews", 268 | "max-results": "20" 269 | }, 270 | "meta": { 271 | "name": "Top Pages (7 Days)", 272 | "description": "Last week's top 20 pages, measured by page views, for all sites." 273 | } 274 | }, 275 | { 276 | "name": "top-pages-30-days", 277 | "frequency": "daily", 278 | "query": { 279 | "dimensions": [ 280 | "ga:hostname", 281 | "ga:pagePath", 282 | "ga:pageTitle" 283 | ], 284 | "metrics": [ 285 | "ga:pageviews", 286 | "ga:sessions", 287 | "ga:users", 288 | "ga:pageviewsPerSession", 289 | "ga:avgSessionDuration", 290 | "ga:exits" 291 | ], 292 | "start-date": "30daysAgo", 293 | "end-date": "yesterday", 294 | "sort": "-ga:pageviews", 295 | "max-results": "20" 296 | }, 297 | "meta": { 298 | "name": "Top Pages (30 Days)", 299 | "description": "Last 30 days' top 20 pages, measured by page views, for all sites." 300 | } 301 | }, 302 | { 303 | "name": "top-domains-7-days", 304 | "frequency": "daily", 305 | "query": { 306 | "dimensions": ["ga:hostname"], 307 | "metrics": ["ga:sessions"], 308 | "start-date": "7daysAgo", 309 | "end-date": "yesterday", 310 | "sort": "-ga:sessions", 311 | "max-results": "20" 312 | }, 313 | "meta": { 314 | "name": "Top Domains (7 Days)", 315 | "description": "Last week's top 20 domains, measured by visits, for all sites." 316 | } 317 | }, 318 | { 319 | "name": "top-domains-30-days", 320 | "frequency": "daily", 321 | "query": { 322 | "dimensions": ["ga:hostname"], 323 | "metrics": ["ga:sessions", "ga:users", "ga:pageviews", "ga:pageviewsPerSession", "ga:avgSessionDuration","ga:exits"], 324 | "start-date": "30daysAgo", 325 | "end-date": "yesterday", 326 | "sort": "-ga:sessions", 327 | "max-results": "20" 328 | }, 329 | "meta": { 330 | "name": "Top Domains (30 Days)", 331 | "description": "Last 30 days' top 20 domains, measured by visits, for all sites." 332 | } 333 | }, 334 | 335 | { 336 | "name": "top-landing-pages-30-days", 337 | "frequency": "daily", 338 | "query": { 339 | "dimensions": ["ga:landingPagePath"], 340 | "metrics": ["ga:sessions", "ga:pageviews", "ga:users", "ga:pageviewsPerSession", "ga:avgSessionDuration", "ga:exits"], 341 | "start-date": "30daysAgo", 342 | "end-date": "yesterday", 343 | "sort": "-ga:sessions", 344 | "max-results": "20" 345 | }, 346 | "meta": { 347 | "name": "Top Landing Pages (30 Days)", 348 | "description": "Last 30 days' Landing Pages, measured by visits, for all sites." 349 | } 350 | }, 351 | 352 | { 353 | "name": "top-traffic-sources-30-days", 354 | "frequency": "daily", 355 | "query": { 356 | "dimensions": ["ga:source", "ga:hasSocialSourceReferral"], 357 | "metrics": ["ga:sessions", "ga:pageviews", "ga:users", "ga:pageviewsPerSession", "ga:avgSessionDuration", "ga:exits"], 358 | "start-date": "30daysAgo", 359 | "end-date": "yesterday", 360 | "sort": "-ga:sessions", 361 | "max-results": "20" 362 | }, 363 | "meta": { 364 | "name": "Top Traffic Sources (30 Days)", 365 | "description": "Last 30 days' Traffic sources, measured by visits, for all sites." 366 | } 367 | }, 368 | 369 | { 370 | "name": "top-exit-pages-30-days", 371 | "frequency": "daily", 372 | "query": { 373 | "dimensions": ["ga:exitPagePath"], 374 | "metrics": ["ga:sessions", "ga:pageviews", "ga:users", "ga:pageviewsPerSession", "ga:avgSessionDuration", "ga:exits"], 375 | "start-date": "30daysAgo", 376 | "end-date": "yesterday", 377 | "sort": "-ga:sessions", 378 | "max-results": "20" 379 | }, 380 | "meta": { 381 | "name": "Top Exit Pages (30 Days)", 382 | "description": "Last 30 days' Exit page paths, measured by visits, for all sites." 383 | } 384 | } 385 | 386 | ] 387 | } 388 | -------------------------------------------------------------------------------- /reports/usa.json: -------------------------------------------------------------------------------- 1 | { 2 | "reports": [ 3 | { 4 | "name": "realtime", 5 | "frequency": "realtime", 6 | "realtime": true, 7 | "query": { 8 | "metrics": ["rt:activeUsers"] 9 | }, 10 | "meta": { 11 | "name": "Active Users Right Now", 12 | "description": "Number of users currently visiting all sites." 13 | } 14 | }, 15 | { 16 | "name": "today", 17 | "frequency": "hourly", 18 | "query": { 19 | "dimensions": ["ga:date", "ga:hour"], 20 | "metrics": ["ga:sessions"], 21 | "start-date": "today", 22 | "end-date": "today" 23 | }, 24 | "meta": { 25 | "name": "Today", 26 | "description": "Today's visits for all sites." 27 | } 28 | }, 29 | { 30 | "name": "devices", 31 | "frequency": "daily", 32 | "slim": true, 33 | "query": { 34 | "dimensions": ["ga:date" ,"ga:deviceCategory"], 35 | "metrics": ["ga:sessions"], 36 | "start-date": "90daysAgo", 37 | "end-date": "yesterday", 38 | "sort": "ga:date" 39 | }, 40 | "meta": { 41 | "name": "Devices", 42 | "description": "90 days of desktop/mobile/tablet visits for all sites." 43 | } 44 | }, 45 | 46 | { 47 | "name": "screen-size", 48 | "frequency": "daily", 49 | "slim": true, 50 | "query": { 51 | "dimensions": ["ga:date" ,"ga:screenResolution"], 52 | "metrics": ["ga:sessions"], 53 | "start-date": "90daysAgo", 54 | "end-date": "yesterday", 55 | "sort": "ga:date", 56 | "filters": ["ga:sessions>1000"] 57 | }, 58 | "meta": { 59 | "name": "Screen Resolution", 60 | "description": "90 days of Screen Resolution visits for all sites. (>5000 sessions)" 61 | } 62 | }, 63 | 64 | { 65 | "name": "language", 66 | "frequency": "daily", 67 | "slim": true, 68 | "query": { 69 | "dimensions": ["ga:date" ,"ga:language"], 70 | "metrics": ["ga:sessions"], 71 | "start-date": "90daysAgo", 72 | "end-date": "yesterday", 73 | "sort": "ga:date", 74 | "filters": ["ga:sessions>10"] 75 | }, 76 | "meta": { 77 | "name": "Browser Language", 78 | "description": "90 days of visits by browser language for all sites. (>1000 sessions)" 79 | } 80 | }, 81 | 82 | 83 | { 84 | "name": "device_model", 85 | "frequency": "daily", 86 | "slim": true, 87 | "query": { 88 | "dimensions": ["ga:date" ,"ga:mobileDeviceModel"], 89 | "metrics": ["ga:sessions"], 90 | "start-date": "90daysAgo", 91 | "end-date": "yesterday", 92 | "sort": "ga:date", 93 | "filters": ["ga:sessions>100"] 94 | }, 95 | "meta": { 96 | "name": "Device Model", 97 | "description": "90 days of visits by Device Model for all sites. (>1000 sessions)" 98 | } 99 | }, 100 | 101 | { 102 | "name": "os", 103 | "frequency": "daily", 104 | "slim": true, 105 | "query": { 106 | "dimensions": ["ga:date" ,"ga:operatingSystem"], 107 | "metrics": ["ga:sessions"], 108 | "start-date": "90daysAgo", 109 | "end-date": "yesterday", 110 | "filters": ["ga:sessions>10"], 111 | "sort": "ga:date" 112 | }, 113 | "meta": { 114 | "name": "Operating Systems", 115 | "description": "90 days of visits, broken down by operating system and date, for all sites. (>100 sessions)" 116 | } 117 | }, 118 | { 119 | "name": "windows", 120 | "frequency": "daily", 121 | "slim": true, 122 | "query": { 123 | "dimensions": ["ga:date" ,"ga:operatingSystemVersion"], 124 | "metrics": ["ga:sessions"], 125 | "start-date": "90daysAgo", 126 | "end-date": "yesterday", 127 | "filters": [ 128 | "ga:operatingSystem==Windows", 129 | "ga:sessions>10" 130 | ], 131 | "sort": "ga:date" 132 | }, 133 | "meta": { 134 | "name": "Windows", 135 | "description": "90 days of visits from Windows users, broken down by operating system version and date, for all sites. (>100 sessions)" 136 | } 137 | }, 138 | { 139 | "name": "browsers", 140 | "frequency": "daily", 141 | "slim": true, 142 | "query": { 143 | "dimensions": ["ga:date" ,"ga:browser"], 144 | "metrics": ["ga:sessions"], 145 | "start-date": "90daysAgo", 146 | "end-date": "yesterday", 147 | "sort": "ga:date,-ga:sessions", 148 | "filters": ["ga:sessions>10"] 149 | }, 150 | "meta": { 151 | "name": "Browsers", 152 | "description": "90 days of visits broken down by browser for all sites. (>100 sessions)" 153 | } 154 | }, 155 | { 156 | "name": "ie", 157 | "frequency": "daily", 158 | "slim": true, 159 | "query": { 160 | "dimensions": ["ga:date","ga:browserVersion"], 161 | "metrics": ["ga:sessions"], 162 | "start-date": "90daysAgo", 163 | "end-date": "yesterday", 164 | "sort": "ga:date,-ga:sessions", 165 | "filters": [ 166 | "ga:browser==Internet Explorer", 167 | "ga:sessions>10" 168 | ] 169 | }, 170 | "meta": { 171 | "name": "Internet Explorer", 172 | "description": "90 days of visits from Internet Explorer users broken down by version for all sites. (>100 sessions)" 173 | } 174 | }, 175 | { 176 | "name": "os-browsers", 177 | "frequency": "daily", 178 | "slim": true, 179 | "query": { 180 | "dimensions": ["ga:date" ,"ga:browser", "ga:operatingSystem"], 181 | "metrics": ["ga:sessions"], 182 | "start-date": "90daysAgo", 183 | "end-date": "yesterday", 184 | "sort": "ga:date,-ga:sessions", 185 | "filters": ["ga:sessions>10"] 186 | }, 187 | "meta": { 188 | "name": "OS-browser combinations", 189 | "description": "90 days of visits broken down by browser and OS for all sites. (>100 sessions)" 190 | } 191 | }, 192 | { 193 | "name": "windows-browsers", 194 | "frequency": "daily", 195 | "slim": true, 196 | "query": { 197 | "dimensions": ["ga:date" ,"ga:browser", "ga:operatingSystemVersion"], 198 | "metrics": ["ga:sessions"], 199 | "start-date": "90daysAgo", 200 | "end-date": "yesterday", 201 | "sort": "ga:date,-ga:sessions", 202 | "filters": [ 203 | "ga:sessions>10", 204 | "ga:operatingSystem==Windows" 205 | ] 206 | }, 207 | "meta": { 208 | "name": "Windows-browser combinations", 209 | "description": "90 days of visits broken down by Windows versions and browser for all sites. (>100 sessions)" 210 | } 211 | }, 212 | { 213 | "name": "windows-ie", 214 | "frequency": "daily", 215 | "slim": true, 216 | "query": { 217 | "dimensions": ["ga:date","ga:browserVersion", "ga:operatingSystemVersion"], 218 | "metrics": ["ga:sessions"], 219 | "start-date": "90daysAgo", 220 | "end-date": "yesterday", 221 | "sort": "ga:date,-ga:sessions", 222 | "filters": [ 223 | "ga:sessions>10", 224 | "ga:browser==Internet Explorer", 225 | "ga:operatingSystem==Windows" 226 | ] 227 | }, 228 | "meta": { 229 | "name": "IE on Windows", 230 | "description": "90 days of visits from IE on Windows broken down by IE and Windows versions for all sites. (>100 sessions)" 231 | } 232 | }, 233 | { 234 | "name": "top-pages-realtime", 235 | "frequency": "realtime", 236 | "realtime": true, 237 | "query": { 238 | "dimensions": ["rt:pagePath", "rt:pageTitle"], 239 | "metrics": ["rt:activeUsers"], 240 | "sort": "-rt:activeUsers", 241 | "max-results": "20" 242 | }, 243 | "meta": { 244 | "name": "Top Pages (Live)", 245 | "description": "The top 20 pages, measured by active onsite users, for all sites." 246 | } 247 | }, 248 | { 249 | "name": "top-domains-7-days", 250 | "frequency": "daily", 251 | "query": { 252 | "dimensions": ["ga:hostname"], 253 | "metrics": ["ga:sessions"], 254 | "start-date": "7daysAgo", 255 | "end-date": "yesterday", 256 | "sort": "-ga:sessions", 257 | "max-results": "20" 258 | }, 259 | "meta": { 260 | "name": "Top Domains (7 Days)", 261 | "description": "Last week's top 20 domains, measured by visits, for all sites." 262 | } 263 | }, 264 | { 265 | "name": "top-domains-30-days", 266 | "frequency": "daily", 267 | "query": { 268 | "dimensions": ["ga:hostname"], 269 | "metrics": ["ga:sessions", "ga:pageviews", "ga:users", "ga:pageviewsPerSession", "ga:avgSessionDuration", "ga:exits"], 270 | "start-date": "30daysAgo", 271 | "end-date": "yesterday", 272 | "sort": "-ga:sessions", 273 | "max-results": "20" 274 | }, 275 | "meta": { 276 | "name": "Top Domains (30 Days)", 277 | "description": "Last 30 days' top 20 domains, measured by visits, for all sites." 278 | } 279 | }, 280 | 281 | { 282 | "name": "top-landing-pages-30-days", 283 | "frequency": "daily", 284 | "query": { 285 | "dimensions": ["ga:landingPagePath"], 286 | "metrics": ["ga:sessions", "ga:pageviews", "ga:users", "ga:pageviewsPerSession", "ga:avgSessionDuration", "ga:exits"], 287 | "start-date": "30daysAgo", 288 | "end-date": "yesterday", 289 | "sort": "-ga:sessions", 290 | "max-results": "20" 291 | }, 292 | "meta": { 293 | "name": "Top Landing Pages (30 Days)", 294 | "description": "Last 30 days' Landing Pages, measured by visits, for all sites." 295 | } 296 | }, 297 | 298 | { 299 | "name": "top-traffic-sources-30-days", 300 | "frequency": "daily", 301 | "query": { 302 | "dimensions": ["ga:source", "ga:hasSocialSourceReferral"], 303 | "metrics": ["ga:sessions", "ga:pageviews", "ga:users", "ga:pageviewsPerSession", "ga:avgSessionDuration", "ga:exits"], 304 | "start-date": "30daysAgo", 305 | "end-date": "yesterday", 306 | "sort": "-ga:sessions", 307 | "max-results": "20" 308 | }, 309 | "meta": { 310 | "name": "Top Traffic Sources (30 Days)", 311 | "description": "Last 30 days' Traffic Sources, measured by visits, for all sites." 312 | } 313 | }, 314 | 315 | { 316 | "name": "top-exit-pages-30-days", 317 | "frequency": "daily", 318 | "query": { 319 | "dimensions": ["ga:exitPagePath"], 320 | "metrics": ["ga:sessions", "ga:pageviews", "ga:users", "ga:pageviewsPerSession", "ga:avgSessionDuration", "ga:exits"], 321 | "start-date": "30daysAgo", 322 | "end-date": "yesterday", 323 | "sort": "-ga:sessions", 324 | "max-results": "20" 325 | }, 326 | "meta": { 327 | "name": "Top Exit Pages (30 Days)", 328 | "description": "Last 30 days' Exit page paths, measured by visits, for all sites." 329 | } 330 | }, 331 | { 332 | "name": "second-level-domains", 333 | "frequency": "daily", 334 | "cut": ["ga:sessions"], 335 | "query": { 336 | "dimensions": ["ga:hostname"], 337 | "metrics": ["ga:sessions"], 338 | "filters": [ 339 | "ga:sessions>4", 340 | "ga:hostname=~^[^\\.]+\\.[^\\.]+$" 341 | ], 342 | "start-date": "14daysAgo", 343 | "end-date": "yesterday", 344 | "sort": "ga:hostname", 345 | "max-results": 10000 346 | }, 347 | "meta": { 348 | "name": "Participating second-level domains.", 349 | "description": "Participating second-level domains over the last 2 weeks." 350 | } 351 | }, 352 | { 353 | "name": "sites", 354 | "frequency": "daily", 355 | "cut": ["ga:sessions"], 356 | "query": { 357 | "dimensions": ["ga:hostname"], 358 | "metrics": ["ga:sessions"], 359 | "filters": [ 360 | "ga:sessions>9" 361 | ], 362 | "start-date": "14daysAgo", 363 | "end-date": "yesterday", 364 | "sort": "ga:hostname", 365 | "max-results": 10000 366 | }, 367 | "meta": { 368 | "name": "Participating hostnames.", 369 | "description": "Participating hostnames over the last 14 days with at least 10 visits." 370 | } 371 | }, 372 | { 373 | "name": "sites-extended", 374 | "frequency": "daily", 375 | "cut": ["ga:sessions"], 376 | "query": { 377 | "dimensions": ["ga:hostname"], 378 | "metrics": ["ga:sessions"], 379 | "filters": [ 380 | "ga:sessions>1" 381 | ], 382 | "start-date": "14daysAgo", 383 | "end-date": "yesterday", 384 | "sort": "ga:hostname", 385 | "max-results": 10000 386 | }, 387 | "meta": { 388 | "name": "Participating hostnames.", 389 | "description": "Participating hostnames over the last 14 days with at least 1 visit." 390 | } 391 | }, 392 | { 393 | "name": "top-downloads-yesterday", 394 | "frequency": "daily", 395 | "query": { 396 | "dimensions": ["ga:pageTitle", "ga:eventLabel", "ga:pagePath"], 397 | "metrics": ["ga:totalEvents"], 398 | "filters": [ 399 | "ga:eventCategory=~ownload", 400 | "ga:pagePath!~(usps.com).*\/(?i)(zip|doc).*" 401 | ], 402 | "start-date": "yesterday", 403 | "end-date": "yesterday", 404 | "sort": "-ga:totalEvents", 405 | "max-results": "100" 406 | }, 407 | "meta": { 408 | "name": "Top Downloads Yesterday", 409 | "description": "Top downloads yesterday" 410 | } 411 | }, 412 | { 413 | "name": "top-downloads-7-days", 414 | "frequency": "daily", 415 | "query": { 416 | "dimensions": ["ga:pageTitle", "ga:eventLabel", "ga:pagePath"], 417 | "metrics": ["ga:totalEvents"], 418 | "filters": [ 419 | "ga:eventCategory=~ownload", 420 | "ga:eventLabel!~swf$", 421 | "ga:pagePath!~(usps.com).*\/(?i)(zip|doc).*" 422 | ], 423 | "start-date": "7daysAgo", 424 | "end-date": "yesterday", 425 | "sort": "-ga:totalEvents", 426 | "max-results": "100" 427 | }, 428 | "meta": { 429 | "name": "Top Downloads (7 Days)", 430 | "description": "Top downloads in the last 7 days." 431 | } 432 | }, 433 | { 434 | "name": "top-cities-realtime", 435 | "frequency": "realtime", 436 | "realtime": true, 437 | "query": { 438 | "dimensions": ["rt:city"], 439 | "metrics": ["rt:activeUsers"], 440 | "sort": "-rt:activeUsers" 441 | }, 442 | "meta": { 443 | "name": "Top Cities (Live)", 444 | "description": "Top cities for active onsite users." 445 | } 446 | }, 447 | { 448 | "name": "top-countries-realtime", 449 | "frequency": "realtime", 450 | "realtime": true, 451 | "query": { 452 | "dimensions": ["rt:country"], 453 | "metrics": ["rt:activeUsers"], 454 | "sort": "-rt:activeUsers" 455 | }, 456 | "meta": { 457 | "name": "Top Cities", 458 | "description": "Top countries for active onsite users." 459 | } 460 | }, 461 | { 462 | "name": "all-pages-realtime", 463 | "frequency": "realtime", 464 | "realtime": true, 465 | "threshold": { 466 | "field": "rt:activeUsers", 467 | "value": "10" 468 | }, 469 | "query": { 470 | "dimensions": ["rt:pagePath", "rt:pageTitle"], 471 | "metrics": ["rt:activeUsers"], 472 | "sort": "-rt:activeUsers", 473 | "max-results": "10000" 474 | }, 475 | "meta": { 476 | "name": "All Pages (Live)", 477 | "description": "Pages, measured by active onsite users, for all sites." 478 | } 479 | }, 480 | { 481 | "name": "all-domains-30-days", 482 | "frequency": "daily", 483 | "query": { 484 | "dimensions": ["ga:hostname"], 485 | "metrics": ["ga:sessions", "ga:pageviews", "ga:users", "ga:pageviewsPerSession", "ga:avgSessionDuration", "ga:exits"], 486 | "start-date": "30daysAgo", 487 | "end-date": "yesterday", 488 | "sort": "-ga:sessions", 489 | "max-results": "10000", 490 | "filters": ["ga:sessions>=10"] 491 | }, 492 | "meta": { 493 | "name": "All Domains (30 Days)", 494 | "description": "Last 30 days' domains, measured by visits, for all sites." 495 | } 496 | } 497 | 498 | ] 499 | } 500 | -------------------------------------------------------------------------------- /test/support/fixtures/results.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | "name": "devices", 3 | "query": { 4 | "start-date": "90daysAgo", 5 | "end-date": "yesterday", 6 | "dimensions": "ga:date,ga:deviceCategory", 7 | "metrics": [ 8 | "ga:sessions" 9 | ], 10 | "sort": [ 11 | "ga:date" 12 | ], 13 | "start-index": 1, 14 | "max-results": 10000, 15 | "samplingLevel": "HIGHER_PRECISION" 16 | }, 17 | "meta": { 18 | "name": "Devices", 19 | "description": "90 days of desktop/mobile/tablet visits for all sites." 20 | }, 21 | "data": [ 22 | { 23 | "date": "2016-11-17", 24 | "device": "desktop", 25 | "visits": "17944716" 26 | }, 27 | { 28 | "date": "2016-11-17", 29 | "device": "mobile", 30 | "visits": "8927140" 31 | }, 32 | { 33 | "date": "2016-11-17", 34 | "device": "tablet", 35 | "visits": "1493615" 36 | }, 37 | { 38 | "date": "2016-11-18", 39 | "device": "desktop", 40 | "visits": "15266887" 41 | }, 42 | { 43 | "date": "2016-11-18", 44 | "device": "mobile", 45 | "visits": "8188620" 46 | }, 47 | { 48 | "date": "2016-11-18", 49 | "device": "tablet", 50 | "visits": "1359252" 51 | }, 52 | { 53 | "date": "2016-11-19", 54 | "device": "desktop", 55 | "visits": "7486523" 56 | }, 57 | { 58 | "date": "2016-11-19", 59 | "device": "mobile", 60 | "visits": "6802302" 61 | }, 62 | { 63 | "date": "2016-11-19", 64 | "device": "tablet", 65 | "visits": "1244910" 66 | }, 67 | { 68 | "date": "2016-11-20", 69 | "device": "desktop", 70 | "visits": "8095419" 71 | }, 72 | { 73 | "date": "2016-11-20", 74 | "device": "mobile", 75 | "visits": "6355972" 76 | }, 77 | { 78 | "date": "2016-11-20", 79 | "device": "tablet", 80 | "visits": "1301498" 81 | }, 82 | { 83 | "date": "2016-11-21", 84 | "device": "desktop", 85 | "visits": "18290260" 86 | }, 87 | { 88 | "date": "2016-11-21", 89 | "device": "mobile", 90 | "visits": "8660823" 91 | }, 92 | { 93 | "date": "2016-11-21", 94 | "device": "tablet", 95 | "visits": "1478005" 96 | }, 97 | { 98 | "date": "2016-11-22", 99 | "device": "desktop", 100 | "visits": "16994015" 101 | }, 102 | { 103 | "date": "2016-11-22", 104 | "device": "mobile", 105 | "visits": "8599485" 106 | }, 107 | { 108 | "date": "2016-11-22", 109 | "device": "tablet", 110 | "visits": "1413091" 111 | }, 112 | { 113 | "date": "2016-11-23", 114 | "device": "desktop", 115 | "visits": "13510470" 116 | }, 117 | { 118 | "date": "2016-11-23", 119 | "device": "mobile", 120 | "visits": "8133319" 121 | }, 122 | { 123 | "date": "2016-11-23", 124 | "device": "tablet", 125 | "visits": "1279496" 126 | }, 127 | { 128 | "date": "2016-11-24", 129 | "device": "desktop", 130 | "visits": "6234988" 131 | }, 132 | { 133 | "date": "2016-11-24", 134 | "device": "mobile", 135 | "visits": "5953655" 136 | }, 137 | { 138 | "date": "2016-11-24", 139 | "device": "tablet", 140 | "visits": "1022314" 141 | }, 142 | { 143 | "date": "2016-11-25", 144 | "device": "desktop", 145 | "visits": "8768054" 146 | }, 147 | { 148 | "date": "2016-11-25", 149 | "device": "mobile", 150 | "visits": "7241617" 151 | }, 152 | { 153 | "date": "2016-11-25", 154 | "device": "tablet", 155 | "visits": "1212316" 156 | }, 157 | { 158 | "date": "2016-11-26", 159 | "device": "desktop", 160 | "visits": "6981808" 161 | }, 162 | { 163 | "date": "2016-11-26", 164 | "device": "mobile", 165 | "visits": "6722048" 166 | }, 167 | { 168 | "date": "2016-11-26", 169 | "device": "tablet", 170 | "visits": "1190519" 171 | }, 172 | { 173 | "date": "2016-11-27", 174 | "device": "desktop", 175 | "visits": "8225314" 176 | }, 177 | { 178 | "date": "2016-11-27", 179 | "device": "mobile", 180 | "visits": "6672403" 181 | }, 182 | { 183 | "date": "2016-11-27", 184 | "device": "tablet", 185 | "visits": "1302649" 186 | }, 187 | { 188 | "date": "2016-11-28", 189 | "device": "desktop", 190 | "visits": "19526901" 191 | }, 192 | { 193 | "date": "2016-11-28", 194 | "device": "mobile", 195 | "visits": "9300099" 196 | }, 197 | { 198 | "date": "2016-11-28", 199 | "device": "tablet", 200 | "visits": "1547016" 201 | }, 202 | { 203 | "date": "2016-11-29", 204 | "device": "desktop", 205 | "visits": "19881628" 206 | }, 207 | { 208 | "date": "2016-11-29", 209 | "device": "mobile", 210 | "visits": "9665025" 211 | }, 212 | { 213 | "date": "2016-11-29", 214 | "device": "tablet", 215 | "visits": "1579273" 216 | }, 217 | { 218 | "date": "2016-11-30", 219 | "device": "desktop", 220 | "visits": "19573065" 221 | }, 222 | { 223 | "date": "2016-11-30", 224 | "device": "mobile", 225 | "visits": "10083858" 226 | }, 227 | { 228 | "date": "2016-11-30", 229 | "device": "tablet", 230 | "visits": "1601741" 231 | }, 232 | { 233 | "date": "2016-12-01", 234 | "device": "desktop", 235 | "visits": "18611610" 236 | }, 237 | { 238 | "date": "2016-12-01", 239 | "device": "mobile", 240 | "visits": "10212056" 241 | }, 242 | { 243 | "date": "2016-12-01", 244 | "device": "tablet", 245 | "visits": "1564647" 246 | }, 247 | { 248 | "date": "2016-12-02", 249 | "device": "desktop", 250 | "visits": "16303740" 251 | }, 252 | { 253 | "date": "2016-12-02", 254 | "device": "mobile", 255 | "visits": "9595214" 256 | }, 257 | { 258 | "date": "2016-12-02", 259 | "device": "tablet", 260 | "visits": "1452885" 261 | }, 262 | { 263 | "date": "2016-12-03", 264 | "device": "desktop", 265 | "visits": "8145522" 266 | }, 267 | { 268 | "date": "2016-12-03", 269 | "device": "mobile", 270 | "visits": "8038915" 271 | }, 272 | { 273 | "date": "2016-12-03", 274 | "device": "tablet", 275 | "visits": "1328963" 276 | }, 277 | { 278 | "date": "2016-12-04", 279 | "device": "desktop", 280 | "visits": "8753097" 281 | }, 282 | { 283 | "date": "2016-12-04", 284 | "device": "mobile", 285 | "visits": "7206951" 286 | }, 287 | { 288 | "date": "2016-12-04", 289 | "device": "tablet", 290 | "visits": "1365981" 291 | }, 292 | { 293 | "date": "2016-12-05", 294 | "device": "desktop", 295 | "visits": "20527426" 296 | }, 297 | { 298 | "date": "2016-12-05", 299 | "device": "mobile", 300 | "visits": "10433381" 301 | }, 302 | { 303 | "date": "2016-12-05", 304 | "device": "tablet", 305 | "visits": "1670167" 306 | }, 307 | { 308 | "date": "2016-12-06", 309 | "device": "desktop", 310 | "visits": "19967407" 311 | }, 312 | { 313 | "date": "2016-12-06", 314 | "device": "mobile", 315 | "visits": "10023434" 316 | }, 317 | { 318 | "date": "2016-12-06", 319 | "device": "tablet", 320 | "visits": "1657519" 321 | }, 322 | { 323 | "date": "2016-12-07", 324 | "device": "desktop", 325 | "visits": "19532055" 326 | }, 327 | { 328 | "date": "2016-12-07", 329 | "device": "mobile", 330 | "visits": "10063789" 331 | }, 332 | { 333 | "date": "2016-12-07", 334 | "device": "tablet", 335 | "visits": "1646568" 336 | }, 337 | { 338 | "date": "2016-12-08", 339 | "device": "desktop", 340 | "visits": "19218012" 341 | }, 342 | { 343 | "date": "2016-12-08", 344 | "device": "mobile", 345 | "visits": "10323528" 346 | }, 347 | { 348 | "date": "2016-12-08", 349 | "device": "tablet", 350 | "visits": "1714556" 351 | }, 352 | { 353 | "date": "2016-12-09", 354 | "device": "desktop", 355 | "visits": "16651672" 356 | }, 357 | { 358 | "date": "2016-12-09", 359 | "device": "mobile", 360 | "visits": "9478158" 361 | }, 362 | { 363 | "date": "2016-12-09", 364 | "device": "tablet", 365 | "visits": "1564344" 366 | }, 367 | { 368 | "date": "2016-12-10", 369 | "device": "desktop", 370 | "visits": "8394504" 371 | }, 372 | { 373 | "date": "2016-12-10", 374 | "device": "mobile", 375 | "visits": "8008296" 376 | }, 377 | { 378 | "date": "2016-12-10", 379 | "device": "tablet", 380 | "visits": "1438817" 381 | }, 382 | { 383 | "date": "2016-12-11", 384 | "device": "desktop", 385 | "visits": "8769674" 386 | }, 387 | { 388 | "date": "2016-12-11", 389 | "device": "mobile", 390 | "visits": "7318707" 391 | }, 392 | { 393 | "date": "2016-12-11", 394 | "device": "tablet", 395 | "visits": "1471781" 396 | }, 397 | { 398 | "date": "2016-12-12", 399 | "device": "desktop", 400 | "visits": "20124799" 401 | }, 402 | { 403 | "date": "2016-12-12", 404 | "device": "mobile", 405 | "visits": "10002557" 406 | }, 407 | { 408 | "date": "2016-12-12", 409 | "device": "tablet", 410 | "visits": "1677637" 411 | }, 412 | { 413 | "date": "2016-12-13", 414 | "device": "desktop", 415 | "visits": "19692582" 416 | }, 417 | { 418 | "date": "2016-12-13", 419 | "device": "mobile", 420 | "visits": "9946246" 421 | }, 422 | { 423 | "date": "2016-12-13", 424 | "device": "tablet", 425 | "visits": "1664839" 426 | }, 427 | { 428 | "date": "2016-12-14", 429 | "device": "desktop", 430 | "visits": "19450673" 431 | }, 432 | { 433 | "date": "2016-12-14", 434 | "device": "mobile", 435 | "visits": "10324397" 436 | }, 437 | { 438 | "date": "2016-12-14", 439 | "device": "tablet", 440 | "visits": "1713116" 441 | }, 442 | { 443 | "date": "2016-12-15", 444 | "device": "desktop", 445 | "visits": "19047361" 446 | }, 447 | { 448 | "date": "2016-12-15", 449 | "device": "mobile", 450 | "visits": "10346150" 451 | }, 452 | { 453 | "date": "2016-12-15", 454 | "device": "tablet", 455 | "visits": "1728800" 456 | }, 457 | { 458 | "date": "2016-12-16", 459 | "device": "desktop", 460 | "visits": "16873358" 461 | }, 462 | { 463 | "date": "2016-12-16", 464 | "device": "mobile", 465 | "visits": "9932215" 466 | }, 467 | { 468 | "date": "2016-12-16", 469 | "device": "tablet", 470 | "visits": "1663874" 471 | }, 472 | { 473 | "date": "2016-12-17", 474 | "device": "desktop", 475 | "visits": "8866860" 476 | }, 477 | { 478 | "date": "2016-12-17", 479 | "device": "mobile", 480 | "visits": "8772502" 481 | }, 482 | { 483 | "date": "2016-12-17", 484 | "device": "tablet", 485 | "visits": "1627369" 486 | }, 487 | { 488 | "date": "2016-12-18", 489 | "device": "desktop", 490 | "visits": "8105408" 491 | }, 492 | { 493 | "date": "2016-12-18", 494 | "device": "mobile", 495 | "visits": "7414904" 496 | }, 497 | { 498 | "date": "2016-12-18", 499 | "device": "tablet", 500 | "visits": "1469536" 501 | }, 502 | { 503 | "date": "2016-12-19", 504 | "device": "desktop", 505 | "visits": "19220918" 506 | }, 507 | { 508 | "date": "2016-12-19", 509 | "device": "mobile", 510 | "visits": "10438620" 511 | }, 512 | { 513 | "date": "2016-12-19", 514 | "device": "tablet", 515 | "visits": "1677447" 516 | }, 517 | { 518 | "date": "2016-12-20", 519 | "device": "desktop", 520 | "visits": "18241079" 521 | }, 522 | { 523 | "date": "2016-12-20", 524 | "device": "mobile", 525 | "visits": "10558487" 526 | }, 527 | { 528 | "date": "2016-12-20", 529 | "device": "tablet", 530 | "visits": "1618781" 531 | }, 532 | { 533 | "date": "2016-12-21", 534 | "device": "desktop", 535 | "visits": "17147953" 536 | }, 537 | { 538 | "date": "2016-12-21", 539 | "device": "mobile", 540 | "visits": "10422959" 541 | }, 542 | { 543 | "date": "2016-12-21", 544 | "device": "tablet", 545 | "visits": "1563992" 546 | }, 547 | { 548 | "date": "2016-12-22", 549 | "device": "desktop", 550 | "visits": "15503945" 551 | }, 552 | { 553 | "date": "2016-12-22", 554 | "device": "mobile", 555 | "visits": "10305992" 556 | }, 557 | { 558 | "date": "2016-12-22", 559 | "device": "tablet", 560 | "visits": "1529405" 561 | }, 562 | { 563 | "date": "2016-12-23", 564 | "device": "desktop", 565 | "visits": "11361437" 566 | }, 567 | { 568 | "date": "2016-12-23", 569 | "device": "mobile", 570 | "visits": "9521278" 571 | }, 572 | { 573 | "date": "2016-12-23", 574 | "device": "tablet", 575 | "visits": "1446075" 576 | }, 577 | { 578 | "date": "2016-12-24", 579 | "device": "desktop", 580 | "visits": "5600182" 581 | }, 582 | { 583 | "date": "2016-12-24", 584 | "device": "mobile", 585 | "visits": "7144987" 586 | }, 587 | { 588 | "date": "2016-12-24", 589 | "device": "tablet", 590 | "visits": "1190168" 591 | }, 592 | { 593 | "date": "2016-12-25", 594 | "device": "desktop", 595 | "visits": "4408666" 596 | }, 597 | { 598 | "date": "2016-12-25", 599 | "device": "mobile", 600 | "visits": "5531137" 601 | }, 602 | { 603 | "date": "2016-12-25", 604 | "device": "tablet", 605 | "visits": "1026063" 606 | }, 607 | { 608 | "date": "2016-12-26", 609 | "device": "desktop", 610 | "visits": "7825098" 611 | }, 612 | { 613 | "date": "2016-12-26", 614 | "device": "mobile", 615 | "visits": "7232890" 616 | }, 617 | { 618 | "date": "2016-12-26", 619 | "device": "tablet", 620 | "visits": "1355893" 621 | }, 622 | { 623 | "date": "2016-12-27", 624 | "device": "desktop", 625 | "visits": "13935273" 626 | }, 627 | { 628 | "date": "2016-12-27", 629 | "device": "mobile", 630 | "visits": "8975892" 631 | }, 632 | { 633 | "date": "2016-12-27", 634 | "device": "tablet", 635 | "visits": "1445369" 636 | }, 637 | { 638 | "date": "2016-12-28", 639 | "device": "desktop", 640 | "visits": "14480665" 641 | }, 642 | { 643 | "date": "2016-12-28", 644 | "device": "mobile", 645 | "visits": "9244411" 646 | }, 647 | { 648 | "date": "2016-12-28", 649 | "device": "tablet", 650 | "visits": "1495648" 651 | }, 652 | { 653 | "date": "2016-12-29", 654 | "device": "desktop", 655 | "visits": "14178667" 656 | }, 657 | { 658 | "date": "2016-12-29", 659 | "device": "mobile", 660 | "visits": "9223986" 661 | }, 662 | { 663 | "date": "2016-12-29", 664 | "device": "tablet", 665 | "visits": "1501026" 666 | }, 667 | { 668 | "date": "2016-12-30", 669 | "device": "desktop", 670 | "visits": "11547674" 671 | }, 672 | { 673 | "date": "2016-12-30", 674 | "device": "mobile", 675 | "visits": "8372061" 676 | }, 677 | { 678 | "date": "2016-12-30", 679 | "device": "tablet", 680 | "visits": "1373276" 681 | }, 682 | { 683 | "date": "2016-12-31", 684 | "device": "desktop", 685 | "visits": "6126765" 686 | }, 687 | { 688 | "date": "2016-12-31", 689 | "device": "mobile", 690 | "visits": "6393735" 691 | }, 692 | { 693 | "date": "2016-12-31", 694 | "device": "tablet", 695 | "visits": "1188851" 696 | }, 697 | { 698 | "date": "2017-01-01", 699 | "device": "desktop", 700 | "visits": "5717572" 701 | }, 702 | { 703 | "date": "2017-01-01", 704 | "device": "mobile", 705 | "visits": "6002253" 706 | }, 707 | { 708 | "date": "2017-01-01", 709 | "device": "tablet", 710 | "visits": "1219702" 711 | }, 712 | { 713 | "date": "2017-01-02", 714 | "device": "desktop", 715 | "visits": "10414034" 716 | }, 717 | { 718 | "date": "2017-01-02", 719 | "device": "mobile", 720 | "visits": "8280913" 721 | }, 722 | { 723 | "date": "2017-01-02", 724 | "device": "tablet", 725 | "visits": "1572182" 726 | }, 727 | { 728 | "date": "2017-01-03", 729 | "device": "desktop", 730 | "visits": "19074040" 731 | }, 732 | { 733 | "date": "2017-01-03", 734 | "device": "mobile", 735 | "visits": "10002388" 736 | }, 737 | { 738 | "date": "2017-01-03", 739 | "device": "tablet", 740 | "visits": "1634073" 741 | }, 742 | { 743 | "date": "2017-01-04", 744 | "device": "desktop", 745 | "visits": "19474263" 746 | }, 747 | { 748 | "date": "2017-01-04", 749 | "device": "mobile", 750 | "visits": "10263370" 751 | }, 752 | { 753 | "date": "2017-01-04", 754 | "device": "tablet", 755 | "visits": "1707684" 756 | }, 757 | { 758 | "date": "2017-01-05", 759 | "device": "desktop", 760 | "visits": "19466017" 761 | }, 762 | { 763 | "date": "2017-01-05", 764 | "device": "mobile", 765 | "visits": "10736442" 766 | }, 767 | { 768 | "date": "2017-01-05", 769 | "device": "tablet", 770 | "visits": "1762507" 771 | }, 772 | { 773 | "date": "2017-01-06", 774 | "device": "desktop", 775 | "visits": "17268777" 776 | }, 777 | { 778 | "date": "2017-01-06", 779 | "device": "mobile", 780 | "visits": "10204089" 781 | }, 782 | { 783 | "date": "2017-01-06", 784 | "device": "tablet", 785 | "visits": "1700304" 786 | }, 787 | { 788 | "date": "2017-01-07", 789 | "device": "desktop", 790 | "visits": "8771825" 791 | }, 792 | { 793 | "date": "2017-01-07", 794 | "device": "mobile", 795 | "visits": "8622569" 796 | }, 797 | { 798 | "date": "2017-01-07", 799 | "device": "tablet", 800 | "visits": "1657525" 801 | }, 802 | { 803 | "date": "2017-01-08", 804 | "device": "desktop", 805 | "visits": "8468167" 806 | }, 807 | { 808 | "date": "2017-01-08", 809 | "device": "mobile", 810 | "visits": "7523797" 811 | }, 812 | { 813 | "date": "2017-01-08", 814 | "device": "tablet", 815 | "visits": "1573548" 816 | }, 817 | { 818 | "date": "2017-01-09", 819 | "device": "desktop", 820 | "visits": "19946515" 821 | }, 822 | { 823 | "date": "2017-01-09", 824 | "device": "mobile", 825 | "visits": "10112103" 826 | }, 827 | { 828 | "date": "2017-01-09", 829 | "device": "tablet", 830 | "visits": "1724557" 831 | }, 832 | { 833 | "date": "2017-01-10", 834 | "device": "desktop", 835 | "visits": "20321640" 836 | }, 837 | { 838 | "date": "2017-01-10", 839 | "device": "mobile", 840 | "visits": "10515776" 841 | }, 842 | { 843 | "date": "2017-01-10", 844 | "device": "tablet", 845 | "visits": "1795632" 846 | }, 847 | { 848 | "date": "2017-01-11", 849 | "device": "desktop", 850 | "visits": "19671577" 851 | }, 852 | { 853 | "date": "2017-01-11", 854 | "device": "mobile", 855 | "visits": "10465313" 856 | }, 857 | { 858 | "date": "2017-01-11", 859 | "device": "tablet", 860 | "visits": "1732368" 861 | }, 862 | { 863 | "date": "2017-01-12", 864 | "device": "desktop", 865 | "visits": "19589937" 866 | }, 867 | { 868 | "date": "2017-01-12", 869 | "device": "mobile", 870 | "visits": "10277052" 871 | }, 872 | { 873 | "date": "2017-01-12", 874 | "device": "tablet", 875 | "visits": "1703584" 876 | }, 877 | { 878 | "date": "2017-01-13", 879 | "device": "desktop", 880 | "visits": "17146743" 881 | }, 882 | { 883 | "date": "2017-01-13", 884 | "device": "mobile", 885 | "visits": "9619211" 886 | }, 887 | { 888 | "date": "2017-01-13", 889 | "device": "tablet", 890 | "visits": "1585216" 891 | }, 892 | { 893 | "date": "2017-01-14", 894 | "device": "desktop", 895 | "visits": "8330783" 896 | }, 897 | { 898 | "date": "2017-01-14", 899 | "device": "mobile", 900 | "visits": "8038168" 901 | }, 902 | { 903 | "date": "2017-01-14", 904 | "device": "tablet", 905 | "visits": "1474055" 906 | }, 907 | { 908 | "date": "2017-01-15", 909 | "device": "desktop", 910 | "visits": "7940108" 911 | }, 912 | { 913 | "date": "2017-01-15", 914 | "device": "mobile", 915 | "visits": "7377663" 916 | }, 917 | { 918 | "date": "2017-01-15", 919 | "device": "tablet", 920 | "visits": "1420365" 921 | }, 922 | { 923 | "date": "2017-01-16", 924 | "device": "desktop", 925 | "visits": "14829426" 926 | }, 927 | { 928 | "date": "2017-01-16", 929 | "device": "mobile", 930 | "visits": "9257283" 931 | }, 932 | { 933 | "date": "2017-01-16", 934 | "device": "tablet", 935 | "visits": "1558470" 936 | }, 937 | { 938 | "date": "2017-01-17", 939 | "device": "desktop", 940 | "visits": "21076771" 941 | }, 942 | { 943 | "date": "2017-01-17", 944 | "device": "mobile", 945 | "visits": "11441390" 946 | }, 947 | { 948 | "date": "2017-01-17", 949 | "device": "tablet", 950 | "visits": "1742698" 951 | }, 952 | { 953 | "date": "2017-01-18", 954 | "device": "desktop", 955 | "visits": "20446130" 956 | }, 957 | { 958 | "date": "2017-01-18", 959 | "device": "mobile", 960 | "visits": "10970693" 961 | }, 962 | { 963 | "date": "2017-01-18", 964 | "device": "tablet", 965 | "visits": "1717717" 966 | }, 967 | { 968 | "date": "2017-01-19", 969 | "device": "desktop", 970 | "visits": "20157052" 971 | }, 972 | { 973 | "date": "2017-01-19", 974 | "device": "mobile", 975 | "visits": "11228989" 976 | }, 977 | { 978 | "date": "2017-01-19", 979 | "device": "tablet", 980 | "visits": "1726224" 981 | }, 982 | { 983 | "date": "2017-01-20", 984 | "device": "desktop", 985 | "visits": "19344217" 986 | }, 987 | { 988 | "date": "2017-01-20", 989 | "device": "mobile", 990 | "visits": "12884804" 991 | }, 992 | { 993 | "date": "2017-01-20", 994 | "device": "tablet", 995 | "visits": "1873116" 996 | }, 997 | { 998 | "date": "2017-01-21", 999 | "device": "desktop", 1000 | "visits": "9950647" 1001 | }, 1002 | { 1003 | "date": "2017-01-21", 1004 | "device": "mobile", 1005 | "visits": "10568161" 1006 | }, 1007 | { 1008 | "date": "2017-01-21", 1009 | "device": "tablet", 1010 | "visits": "1783297" 1011 | }, 1012 | { 1013 | "date": "2017-01-22", 1014 | "device": "desktop", 1015 | "visits": "10151644" 1016 | }, 1017 | { 1018 | "date": "2017-01-22", 1019 | "device": "mobile", 1020 | "visits": "9316374" 1021 | }, 1022 | { 1023 | "date": "2017-01-22", 1024 | "device": "tablet", 1025 | "visits": "1822457" 1026 | }, 1027 | { 1028 | "date": "2017-01-23", 1029 | "device": "desktop", 1030 | "visits": "23257771" 1031 | }, 1032 | { 1033 | "date": "2017-01-23", 1034 | "device": "mobile", 1035 | "visits": "12281874" 1036 | }, 1037 | { 1038 | "date": "2017-01-23", 1039 | "device": "tablet", 1040 | "visits": "1957768" 1041 | }, 1042 | { 1043 | "date": "2017-01-24", 1044 | "device": "desktop", 1045 | "visits": "21802654" 1046 | }, 1047 | { 1048 | "date": "2017-01-24", 1049 | "device": "mobile", 1050 | "visits": "11787571" 1051 | }, 1052 | { 1053 | "date": "2017-01-24", 1054 | "device": "tablet", 1055 | "visits": "1840512" 1056 | }, 1057 | { 1058 | "date": "2017-01-25", 1059 | "device": "desktop", 1060 | "visits": "21217961" 1061 | }, 1062 | { 1063 | "date": "2017-01-25", 1064 | "device": "mobile", 1065 | "visits": "12259488" 1066 | }, 1067 | { 1068 | "date": "2017-01-25", 1069 | "device": "tablet", 1070 | "visits": "1824556" 1071 | }, 1072 | { 1073 | "date": "2017-01-26", 1074 | "device": "desktop", 1075 | "visits": "20151178" 1076 | }, 1077 | { 1078 | "date": "2017-01-26", 1079 | "device": "mobile", 1080 | "visits": "11692776" 1081 | }, 1082 | { 1083 | "date": "2017-01-26", 1084 | "device": "tablet", 1085 | "visits": "1720242" 1086 | }, 1087 | { 1088 | "date": "2017-01-27", 1089 | "device": "desktop", 1090 | "visits": "17657726" 1091 | }, 1092 | { 1093 | "date": "2017-01-27", 1094 | "device": "mobile", 1095 | "visits": "10761667" 1096 | }, 1097 | { 1098 | "date": "2017-01-27", 1099 | "device": "tablet", 1100 | "visits": "1574402" 1101 | }, 1102 | { 1103 | "date": "2017-01-28", 1104 | "device": "desktop", 1105 | "visits": "9175780" 1106 | }, 1107 | { 1108 | "date": "2017-01-28", 1109 | "device": "mobile", 1110 | "visits": "9316210" 1111 | }, 1112 | { 1113 | "date": "2017-01-28", 1114 | "device": "tablet", 1115 | "visits": "1486173" 1116 | }, 1117 | { 1118 | "date": "2017-01-29", 1119 | "device": "desktop", 1120 | "visits": "9761406" 1121 | }, 1122 | { 1123 | "date": "2017-01-29", 1124 | "device": "mobile", 1125 | "visits": "9702597" 1126 | }, 1127 | { 1128 | "date": "2017-01-29", 1129 | "device": "tablet", 1130 | "visits": "1606222" 1131 | }, 1132 | { 1133 | "date": "2017-01-30", 1134 | "device": "desktop", 1135 | "visits": "22638067" 1136 | }, 1137 | { 1138 | "date": "2017-01-30", 1139 | "device": "mobile", 1140 | "visits": "12653369" 1141 | }, 1142 | { 1143 | "date": "2017-01-30", 1144 | "device": "tablet", 1145 | "visits": "1858651" 1146 | }, 1147 | { 1148 | "date": "2017-01-31", 1149 | "device": "desktop", 1150 | "visits": "22251428" 1151 | }, 1152 | { 1153 | "date": "2017-01-31", 1154 | "device": "mobile", 1155 | "visits": "12268125" 1156 | }, 1157 | { 1158 | "date": "2017-01-31", 1159 | "device": "tablet", 1160 | "visits": "1819209" 1161 | }, 1162 | { 1163 | "date": "2017-02-01", 1164 | "device": "desktop", 1165 | "visits": "21087290" 1166 | }, 1167 | { 1168 | "date": "2017-02-01", 1169 | "device": "mobile", 1170 | "visits": "12257163" 1171 | }, 1172 | { 1173 | "date": "2017-02-01", 1174 | "device": "tablet", 1175 | "visits": "1791769" 1176 | }, 1177 | { 1178 | "date": "2017-02-02", 1179 | "device": "desktop", 1180 | "visits": "20524207" 1181 | }, 1182 | { 1183 | "date": "2017-02-02", 1184 | "device": "mobile", 1185 | "visits": "12114547" 1186 | }, 1187 | { 1188 | "date": "2017-02-02", 1189 | "device": "tablet", 1190 | "visits": "1757504" 1191 | }, 1192 | { 1193 | "date": "2017-02-03", 1194 | "device": "desktop", 1195 | "visits": "17997793" 1196 | }, 1197 | { 1198 | "date": "2017-02-03", 1199 | "device": "mobile", 1200 | "visits": "11483512" 1201 | }, 1202 | { 1203 | "date": "2017-02-03", 1204 | "device": "tablet", 1205 | "visits": "1646621" 1206 | }, 1207 | { 1208 | "date": "2017-02-04", 1209 | "device": "desktop", 1210 | "visits": "9313172" 1211 | }, 1212 | { 1213 | "date": "2017-02-04", 1214 | "device": "mobile", 1215 | "visits": "9544262" 1216 | }, 1217 | { 1218 | "date": "2017-02-04", 1219 | "device": "tablet", 1220 | "visits": "1503310" 1221 | }, 1222 | { 1223 | "date": "2017-02-05", 1224 | "device": "desktop", 1225 | "visits": "8833525" 1226 | }, 1227 | { 1228 | "date": "2017-02-05", 1229 | "device": "mobile", 1230 | "visits": "8273273" 1231 | }, 1232 | { 1233 | "date": "2017-02-05", 1234 | "device": "tablet", 1235 | "visits": "1436846" 1236 | }, 1237 | { 1238 | "date": "2017-02-06", 1239 | "device": "desktop", 1240 | "visits": "21775734" 1241 | }, 1242 | { 1243 | "date": "2017-02-06", 1244 | "device": "mobile", 1245 | "visits": "12223955" 1246 | }, 1247 | { 1248 | "date": "2017-02-06", 1249 | "device": "tablet", 1250 | "visits": "1821893" 1251 | }, 1252 | { 1253 | "date": "2017-02-07", 1254 | "device": "desktop", 1255 | "visits": "22100599" 1256 | }, 1257 | { 1258 | "date": "2017-02-07", 1259 | "device": "mobile", 1260 | "visits": "12625240" 1261 | }, 1262 | { 1263 | "date": "2017-02-07", 1264 | "device": "tablet", 1265 | "visits": "1899859" 1266 | }, 1267 | { 1268 | "date": "2017-02-08", 1269 | "device": "desktop", 1270 | "visits": "22031758" 1271 | }, 1272 | { 1273 | "date": "2017-02-08", 1274 | "device": "mobile", 1275 | "visits": "13262193" 1276 | }, 1277 | { 1278 | "date": "2017-02-08", 1279 | "device": "tablet", 1280 | "visits": "1931228" 1281 | }, 1282 | { 1283 | "date": "2017-02-09", 1284 | "device": "desktop", 1285 | "visits": "20575032" 1286 | }, 1287 | { 1288 | "date": "2017-02-09", 1289 | "device": "mobile", 1290 | "visits": "12979335" 1291 | }, 1292 | { 1293 | "date": "2017-02-09", 1294 | "device": "tablet", 1295 | "visits": "1921387" 1296 | }, 1297 | { 1298 | "date": "2017-02-10", 1299 | "device": "desktop", 1300 | "visits": "17711813" 1301 | }, 1302 | { 1303 | "date": "2017-02-10", 1304 | "device": "mobile", 1305 | "visits": "11965905" 1306 | }, 1307 | { 1308 | "date": "2017-02-10", 1309 | "device": "tablet", 1310 | "visits": "1675788" 1311 | }, 1312 | { 1313 | "date": "2017-02-11", 1314 | "device": "desktop", 1315 | "visits": "9097741" 1316 | }, 1317 | { 1318 | "date": "2017-02-11", 1319 | "device": "mobile", 1320 | "visits": "10059393" 1321 | }, 1322 | { 1323 | "date": "2017-02-11", 1324 | "device": "tablet", 1325 | "visits": "1542236" 1326 | }, 1327 | { 1328 | "date": "2017-02-12", 1329 | "device": "desktop", 1330 | "visits": "9652936" 1331 | }, 1332 | { 1333 | "date": "2017-02-12", 1334 | "device": "mobile", 1335 | "visits": "9133410" 1336 | }, 1337 | { 1338 | "date": "2017-02-12", 1339 | "device": "tablet", 1340 | "visits": "1592009" 1341 | }, 1342 | { 1343 | "date": "2017-02-13", 1344 | "device": "desktop", 1345 | "visits": "20780584" 1346 | }, 1347 | { 1348 | "date": "2017-02-13", 1349 | "device": "mobile", 1350 | "visits": "12435261" 1351 | }, 1352 | { 1353 | "date": "2017-02-13", 1354 | "device": "tablet", 1355 | "visits": "1753516" 1356 | }, 1357 | { 1358 | "date": "2017-02-14", 1359 | "device": "desktop", 1360 | "visits": "19207139" 1361 | }, 1362 | { 1363 | "date": "2017-02-14", 1364 | "device": "mobile", 1365 | "visits": "11879814" 1366 | }, 1367 | { 1368 | "date": "2017-02-14", 1369 | "device": "tablet", 1370 | "visits": "1642179" 1371 | } 1372 | ], 1373 | "totals": { 1374 | "visits": 2380289500, 1375 | "devices": { 1376 | "desktop": 1369555309, 1377 | "mobile": 868783942, 1378 | "tablet": 141950249 1379 | } 1380 | }, 1381 | "taken_at": "2017-02-15T15:44:53.044Z" 1382 | } 1383 | 1384 | --------------------------------------------------------------------------------