├── .editorconfig ├── .env-dist ├── .eslintignore ├── .eslintrc.js ├── .github └── workflows │ └── ci_workflow.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── config └── datasets │ ├── hardware.json │ ├── usage-behavior.json │ └── user-activity.json ├── docs └── example-redash-config.json ├── package-lock.json ├── package.json └── src ├── formatters ├── BabbageFormatter.js ├── Formatter.js ├── QuantumFormatter.js └── RedashFormatter.js ├── index.js ├── processData.js └── tests ├── special └── api-equivalence.test.js ├── standard └── summary.test.js └── utils.js /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | # Unix line endings 5 | end_of_line = lf 6 | insert_final_newline = true 7 | 8 | trim_trailing_whitespace = true 9 | 10 | [*.{py,html,css,styl,js,json,md}] 11 | indent_style = space 12 | indent_size = 4 13 | 14 | [*.yml] 15 | indent_style = space 16 | indent_size = 2 17 | -------------------------------------------------------------------------------- /.env-dist: -------------------------------------------------------------------------------- 1 | AWS_BUCKET_NAME= 2 | AWS_REGION= 3 | AWS_ACCESS_KEY_ID= 4 | AWS_SECRET_ACCESS_KEY= 5 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | package-lock.json 2 | 3 | # Reverse matches (i.e., DO NOT ignore these files) 4 | !.eslintrc.js 5 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | node: true, 4 | mocha: true, 5 | }, 6 | extends: [ 7 | 'eslint:recommended', 8 | 'plugin:node/recommended', 9 | ], 10 | plugins: [ 11 | 'json', 12 | 'node', 13 | 'mocha', 14 | ], 15 | root: true, 16 | rules: { 17 | // Disabled 18 | 'node/no-unpublished-require': 0, 19 | 20 | // Errors 21 | 'eqeqeq': 'error', 22 | 23 | // Warnings 24 | 'no-console': 'warn', 25 | 26 | // Stylistic warnings 27 | 'quotes': ['warn', 'single', { avoidEscape: true }], 28 | 'indent': ['warn', 4, { SwitchCase: 1 }], 29 | }, 30 | }; 31 | -------------------------------------------------------------------------------- /.github/workflows/ci_workflow.yml: -------------------------------------------------------------------------------- 1 | name: CI Workflow 2 | on: 3 | push: 4 | 5 | jobs: 6 | test: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - name: Checkout code 10 | uses: actions/checkout@v4 11 | with: 12 | persist-credentials: false 13 | - name: Check code formatting 14 | run: make lint 15 | - name: Run unit tests 16 | run: | 17 | touch .env 18 | make test 19 | deploy: 20 | runs-on: ubuntu-latest 21 | if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/') 22 | permissions: 23 | id-token: write 24 | contents: read 25 | needs: 26 | - test 27 | steps: 28 | - name: Checkout code 29 | uses: actions/checkout@v4 30 | with: 31 | persist-credentials: false 32 | - name: Set Docker tag 33 | id: docker_tag 34 | env: 35 | BRANCH: ${{ github.head_ref }} 36 | run: | 37 | if [[ "$BRANCH" == refs/tags/* ]]; then 38 | echo "tag=$BRANCH" >> $GITHUB_OUTPUT 39 | else 40 | echo "tag=latest" >> $GITHUB_OUTPUT 41 | fi 42 | - name: Build the Docker image 43 | env: 44 | DOCKER_TAG: ${{ steps.docker_tag.outputs.tag }} 45 | run: docker build . -t us-docker.pkg.dev/moz-fx-data-artifacts-prod/ensemble-transposer/ensemble-transposer:$DOCKER_TAG 46 | - name: Push Docker image to GAR 47 | uses: mozilla-it/deploy-actions/docker-push@v4.3.2 48 | with: 49 | project_id: moz-fx-data-artifacts-prod 50 | image_tags: us-docker.pkg.dev/moz-fx-data-artifacts-prod/ensemble-transposer/ensemble-transposer:${{ steps.docker_tag.outputs.tag }} 51 | workload_identity_pool_project_number: ${{ vars.GCPV2_WORKLOAD_IDENTITY_POOL_PROJECT_NUMBER }} 52 | service_account_name: ensemble-transposer 53 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | 3 | # Misc 4 | .DS_Store 5 | *.sw? 6 | 7 | # Node 8 | node_modules 9 | npm-debug.log* 10 | *.log 11 | 12 | # output 13 | target 14 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Community Participation Guidelines 2 | 3 | This repository is governed by Mozilla's code of conduct and etiquette guidelines. 4 | For more details, please read the 5 | [Mozilla Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/). 6 | 7 | ## How to Report 8 | For more information on how to report violations of the Community Participation Guidelines, please read our '[How to Report](https://www.mozilla.org/about/governance/policies/participation/reporting/)' page. 9 | 10 | 16 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:10.18.1 2 | 3 | WORKDIR /app 4 | 5 | # Install node requirements and clean up unneeded cache data 6 | COPY package*.json ./ 7 | RUN npm install && \ 8 | npm cache clear --force && \ 9 | rm -rf ~app/.node-gyp 10 | 11 | # Copy all other needed files into the image 12 | # 13 | # NB: Docker copies the contents of directories, not the directories themselves, 14 | # to the specified target. That's why we need to name target directories. 15 | COPY src ./src 16 | COPY config ./config 17 | COPY .eslintignore .eslintrc.js ./ 18 | 19 | CMD ["npm", "start"] 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Mozilla Public License Version 2.0 2 | ================================== 3 | 4 | 1. Definitions 5 | -------------- 6 | 7 | 1.1. "Contributor" 8 | means each individual or legal entity that creates, contributes to 9 | the creation of, or owns Covered Software. 10 | 11 | 1.2. "Contributor Version" 12 | means the combination of the Contributions of others (if any) used 13 | by a Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | means Covered Software of a particular Contributor. 17 | 18 | 1.4. "Covered Software" 19 | means Source Code Form to which the initial Contributor has attached 20 | the notice in Exhibit A, the Executable Form of such Source Code 21 | Form, and Modifications of such Source Code Form, in each case 22 | including portions thereof. 23 | 24 | 1.5. "Incompatible With Secondary Licenses" 25 | means 26 | 27 | (a) that the initial Contributor has attached the notice described 28 | in Exhibit B to the Covered Software; or 29 | 30 | (b) that the Covered Software was made available under the terms of 31 | version 1.1 or earlier of the License, but not also under the 32 | terms of a Secondary License. 33 | 34 | 1.6. "Executable Form" 35 | means any form of the work other than Source Code Form. 36 | 37 | 1.7. "Larger Work" 38 | means a work that combines Covered Software with other material, in 39 | a separate file or files, that is not Covered Software. 40 | 41 | 1.8. "License" 42 | means this document. 43 | 44 | 1.9. "Licensable" 45 | means having the right to grant, to the maximum extent possible, 46 | whether at the time of the initial grant or subsequently, any and 47 | all of the rights conveyed by this License. 48 | 49 | 1.10. "Modifications" 50 | means any of the following: 51 | 52 | (a) any file in Source Code Form that results from an addition to, 53 | deletion from, or modification of the contents of Covered 54 | Software; or 55 | 56 | (b) any new file in Source Code Form that contains any Covered 57 | Software. 58 | 59 | 1.11. "Patent Claims" of a Contributor 60 | means any patent claim(s), including without limitation, method, 61 | process, and apparatus claims, in any patent Licensable by such 62 | Contributor that would be infringed, but for the grant of the 63 | License, by the making, using, selling, offering for sale, having 64 | made, import, or transfer of either its Contributions or its 65 | Contributor Version. 66 | 67 | 1.12. "Secondary License" 68 | means either the GNU General Public License, Version 2.0, the GNU 69 | Lesser General Public License, Version 2.1, the GNU Affero General 70 | Public License, Version 3.0, or any later versions of those 71 | licenses. 72 | 73 | 1.13. "Source Code Form" 74 | means the form of the work preferred for making modifications. 75 | 76 | 1.14. "You" (or "Your") 77 | means an individual or a legal entity exercising rights under this 78 | License. For legal entities, "You" includes any entity that 79 | controls, is controlled by, or is under common control with You. For 80 | purposes of this definition, "control" means (a) the power, direct 81 | or indirect, to cause the direction or management of such entity, 82 | whether by contract or otherwise, or (b) ownership of more than 83 | fifty percent (50%) of the outstanding shares or beneficial 84 | ownership of such entity. 85 | 86 | 2. License Grants and Conditions 87 | -------------------------------- 88 | 89 | 2.1. Grants 90 | 91 | Each Contributor hereby grants You a world-wide, royalty-free, 92 | non-exclusive license: 93 | 94 | (a) under intellectual property rights (other than patent or trademark) 95 | Licensable by such Contributor to use, reproduce, make available, 96 | modify, display, perform, distribute, and otherwise exploit its 97 | Contributions, either on an unmodified basis, with Modifications, or 98 | as part of a Larger Work; and 99 | 100 | (b) under Patent Claims of such Contributor to make, use, sell, offer 101 | for sale, have made, import, and otherwise transfer either its 102 | Contributions or its Contributor Version. 103 | 104 | 2.2. Effective Date 105 | 106 | The licenses granted in Section 2.1 with respect to any Contribution 107 | become effective for each Contribution on the date the Contributor first 108 | distributes such Contribution. 109 | 110 | 2.3. Limitations on Grant Scope 111 | 112 | The licenses granted in this Section 2 are the only rights granted under 113 | this License. No additional rights or licenses will be implied from the 114 | distribution or licensing of Covered Software under this License. 115 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 116 | Contributor: 117 | 118 | (a) for any code that a Contributor has removed from Covered Software; 119 | or 120 | 121 | (b) for infringements caused by: (i) Your and any other third party's 122 | modifications of Covered Software, or (ii) the combination of its 123 | Contributions with other software (except as part of its Contributor 124 | Version); or 125 | 126 | (c) under Patent Claims infringed by Covered Software in the absence of 127 | its Contributions. 128 | 129 | This License does not grant any rights in the trademarks, service marks, 130 | or logos of any Contributor (except as may be necessary to comply with 131 | the notice requirements in Section 3.4). 132 | 133 | 2.4. Subsequent Licenses 134 | 135 | No Contributor makes additional grants as a result of Your choice to 136 | distribute the Covered Software under a subsequent version of this 137 | License (see Section 10.2) or under the terms of a Secondary License (if 138 | permitted under the terms of Section 3.3). 139 | 140 | 2.5. Representation 141 | 142 | Each Contributor represents that the Contributor believes its 143 | Contributions are its original creation(s) or it has sufficient rights 144 | to grant the rights to its Contributions conveyed by this License. 145 | 146 | 2.6. Fair Use 147 | 148 | This License is not intended to limit any rights You have under 149 | applicable copyright doctrines of fair use, fair dealing, or other 150 | equivalents. 151 | 152 | 2.7. Conditions 153 | 154 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 155 | in Section 2.1. 156 | 157 | 3. Responsibilities 158 | ------------------- 159 | 160 | 3.1. Distribution of Source Form 161 | 162 | All distribution of Covered Software in Source Code Form, including any 163 | Modifications that You create or to which You contribute, must be under 164 | the terms of this License. You must inform recipients that the Source 165 | Code Form of the Covered Software is governed by the terms of this 166 | License, and how they can obtain a copy of this License. You may not 167 | attempt to alter or restrict the recipients' rights in the Source Code 168 | Form. 169 | 170 | 3.2. Distribution of Executable Form 171 | 172 | If You distribute Covered Software in Executable Form then: 173 | 174 | (a) such Covered Software must also be made available in Source Code 175 | Form, as described in Section 3.1, and You must inform recipients of 176 | the Executable Form how they can obtain a copy of such Source Code 177 | Form by reasonable means in a timely manner, at a charge no more 178 | than the cost of distribution to the recipient; and 179 | 180 | (b) You may distribute such Executable Form under the terms of this 181 | License, or sublicense it under different terms, provided that the 182 | license for the Executable Form does not attempt to limit or alter 183 | the recipients' rights in the Source Code Form under this License. 184 | 185 | 3.3. Distribution of a Larger Work 186 | 187 | You may create and distribute a Larger Work under terms of Your choice, 188 | provided that You also comply with the requirements of this License for 189 | the Covered Software. If the Larger Work is a combination of Covered 190 | Software with a work governed by one or more Secondary Licenses, and the 191 | Covered Software is not Incompatible With Secondary Licenses, this 192 | License permits You to additionally distribute such Covered Software 193 | under the terms of such Secondary License(s), so that the recipient of 194 | the Larger Work may, at their option, further distribute the Covered 195 | Software under the terms of either this License or such Secondary 196 | License(s). 197 | 198 | 3.4. Notices 199 | 200 | You may not remove or alter the substance of any license notices 201 | (including copyright notices, patent notices, disclaimers of warranty, 202 | or limitations of liability) contained within the Source Code Form of 203 | the Covered Software, except that You may alter any license notices to 204 | the extent required to remedy known factual inaccuracies. 205 | 206 | 3.5. Application of Additional Terms 207 | 208 | You may choose to offer, and to charge a fee for, warranty, support, 209 | indemnity or liability obligations to one or more recipients of Covered 210 | Software. However, You may do so only on Your own behalf, and not on 211 | behalf of any Contributor. You must make it absolutely clear that any 212 | such warranty, support, indemnity, or liability obligation is offered by 213 | You alone, and You hereby agree to indemnify every Contributor for any 214 | liability incurred by such Contributor as a result of warranty, support, 215 | indemnity or liability terms You offer. You may include additional 216 | disclaimers of warranty and limitations of liability specific to any 217 | jurisdiction. 218 | 219 | 4. Inability to Comply Due to Statute or Regulation 220 | --------------------------------------------------- 221 | 222 | If it is impossible for You to comply with any of the terms of this 223 | License with respect to some or all of the Covered Software due to 224 | statute, judicial order, or regulation then You must: (a) comply with 225 | the terms of this License to the maximum extent possible; and (b) 226 | describe the limitations and the code they affect. Such description must 227 | be placed in a text file included with all distributions of the Covered 228 | Software under this License. Except to the extent prohibited by statute 229 | or regulation, such description must be sufficiently detailed for a 230 | recipient of ordinary skill to be able to understand it. 231 | 232 | 5. Termination 233 | -------------- 234 | 235 | 5.1. The rights granted under this License will terminate automatically 236 | if You fail to comply with any of its terms. However, if You become 237 | compliant, then the rights granted under this License from a particular 238 | Contributor are reinstated (a) provisionally, unless and until such 239 | Contributor explicitly and finally terminates Your grants, and (b) on an 240 | ongoing basis, if such Contributor fails to notify You of the 241 | non-compliance by some reasonable means prior to 60 days after You have 242 | come back into compliance. Moreover, Your grants from a particular 243 | Contributor are reinstated on an ongoing basis if such Contributor 244 | notifies You of the non-compliance by some reasonable means, this is the 245 | first time You have received notice of non-compliance with this License 246 | from such Contributor, and You become compliant prior to 30 days after 247 | Your receipt of the notice. 248 | 249 | 5.2. If You initiate litigation against any entity by asserting a patent 250 | infringement claim (excluding declaratory judgment actions, 251 | counter-claims, and cross-claims) alleging that a Contributor Version 252 | directly or indirectly infringes any patent, then the rights granted to 253 | You by any and all Contributors for the Covered Software under Section 254 | 2.1 of this License shall terminate. 255 | 256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 257 | end user license agreements (excluding distributors and resellers) which 258 | have been validly granted by You or Your distributors under this License 259 | prior to termination shall survive termination. 260 | 261 | ************************************************************************ 262 | * * 263 | * 6. Disclaimer of Warranty * 264 | * ------------------------- * 265 | * * 266 | * Covered Software is provided under this License on an "as is" * 267 | * basis, without warranty of any kind, either expressed, implied, or * 268 | * statutory, including, without limitation, warranties that the * 269 | * Covered Software is free of defects, merchantable, fit for a * 270 | * particular purpose or non-infringing. The entire risk as to the * 271 | * quality and performance of the Covered Software is with You. * 272 | * Should any Covered Software prove defective in any respect, You * 273 | * (not any Contributor) assume the cost of any necessary servicing, * 274 | * repair, or correction. This disclaimer of warranty constitutes an * 275 | * essential part of this License. No use of any Covered Software is * 276 | * authorized under this License except under this disclaimer. * 277 | * * 278 | ************************************************************************ 279 | 280 | ************************************************************************ 281 | * * 282 | * 7. Limitation of Liability * 283 | * -------------------------- * 284 | * * 285 | * Under no circumstances and under no legal theory, whether tort * 286 | * (including negligence), contract, or otherwise, shall any * 287 | * Contributor, or anyone who distributes Covered Software as * 288 | * permitted above, be liable to You for any direct, indirect, * 289 | * special, incidental, or consequential damages of any character * 290 | * including, without limitation, damages for lost profits, loss of * 291 | * goodwill, work stoppage, computer failure or malfunction, or any * 292 | * and all other commercial damages or losses, even if such party * 293 | * shall have been informed of the possibility of such damages. This * 294 | * limitation of liability shall not apply to liability for death or * 295 | * personal injury resulting from such party's negligence to the * 296 | * extent applicable law prohibits such limitation. Some * 297 | * jurisdictions do not allow the exclusion or limitation of * 298 | * incidental or consequential damages, so this exclusion and * 299 | * limitation may not apply to You. * 300 | * * 301 | ************************************************************************ 302 | 303 | 8. Litigation 304 | ------------- 305 | 306 | Any litigation relating to this License may be brought only in the 307 | courts of a jurisdiction where the defendant maintains its principal 308 | place of business and such litigation shall be governed by laws of that 309 | jurisdiction, without reference to its conflict-of-law provisions. 310 | Nothing in this Section shall prevent a party's ability to bring 311 | cross-claims or counter-claims. 312 | 313 | 9. Miscellaneous 314 | ---------------- 315 | 316 | This License represents the complete agreement concerning the subject 317 | matter hereof. If any provision of this License is held to be 318 | unenforceable, such provision shall be reformed only to the extent 319 | necessary to make it enforceable. Any law or regulation which provides 320 | that the language of a contract shall be construed against the drafter 321 | shall not be used to construe this License against a Contributor. 322 | 323 | 10. Versions of the License 324 | --------------------------- 325 | 326 | 10.1. New Versions 327 | 328 | Mozilla Foundation is the license steward. Except as provided in Section 329 | 10.3, no one other than the license steward has the right to modify or 330 | publish new versions of this License. Each version will be given a 331 | distinguishing version number. 332 | 333 | 10.2. Effect of New Versions 334 | 335 | You may distribute the Covered Software under the terms of the version 336 | of the License under which You originally received the Covered Software, 337 | or under the terms of any subsequent version published by the license 338 | steward. 339 | 340 | 10.3. Modified Versions 341 | 342 | If you create software not governed by this License, and you want to 343 | create a new license for such software, you may create and use a 344 | modified version of this License if you rename the license and remove 345 | any references to the name of the license steward (except to note that 346 | such modified license differs from this License). 347 | 348 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 349 | Licenses 350 | 351 | If You choose to distribute Source Code Form that is Incompatible With 352 | Secondary Licenses under the terms of this version of the License, the 353 | notice described in Exhibit B of this License must be attached. 354 | 355 | Exhibit A - Source Code Form License Notice 356 | ------------------------------------------- 357 | 358 | This Source Code Form is subject to the terms of the Mozilla Public 359 | License, v. 2.0. If a copy of the MPL was not distributed with this 360 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 361 | 362 | If it is not possible or desirable to put the notice in a particular 363 | file, then You may include the notice in a location (such as a LICENSE 364 | file in a relevant directory) where a recipient would be likely to look 365 | for such a notice. 366 | 367 | You may add additional accurate notices of copyright ownership. 368 | 369 | Exhibit B - "Incompatible With Secondary Licenses" Notice 370 | --------------------------------------------------------- 371 | 372 | This Source Code Form is "Incompatible With Secondary Licenses", as 373 | defined by the Mozilla Public License, v. 2.0. 374 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: build start lint test compare shell 2 | 3 | help: 4 | @echo "Makefile commands for local development:" 5 | @echo 6 | @echo " build Build the Docker image" 7 | @echo " start Run ensemble-transposer" 8 | @echo " lint Lint source code" 9 | @echo " test Run tests" 10 | @echo " compare Compare development output to production output" 11 | @echo " shell Start a Bash shell" 12 | 13 | build: 14 | docker image build --tag ensemble-transposer . 15 | 16 | start: build 17 | docker container run --rm --tty --env-file=.env ensemble-transposer npm start 18 | 19 | lint: build 20 | docker container run --rm --tty ensemble-transposer npm run lint 21 | 22 | test: build 23 | docker container run --rm --tty --env-file=.env ensemble-transposer npm test 24 | 25 | compare: build 26 | docker container run --rm --tty --env-file=.env \ 27 | ensemble-transposer npm run compare 28 | 29 | shell: 30 | docker container run --rm --tty --interactive --env-file=.env \ 31 | ensemble-transposer /bin/bash 32 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ensemble-transposer re-formats existing data so that it can be used by the 2 | [Firefox Public Data Report](https://data.firefox.com). 3 | 4 | Mozilla already publishes raw data: numbers and identifiers. That's great, but 5 | it can be difficult to work with. ensemble-transposer takes that raw data, 6 | organizes it, adds useful information like explanations, and generates a series 7 | of files that are much easier for developers to work with. 8 | [Ensemble](https://github.com/mozilla/ensemble), the platform that powers the 9 | Firefox Public Data Report, uses this improved and re-formatted data to build 10 | dashboards. 11 | 12 | Other applications are also welcome to use the data that ensemble-transposer 13 | outputs. See the [API documentation](#API) for more information. 14 | 15 | ensemble-transposer can easily enhance any data that adheres to [this 16 | format](https://public-data.telemetry.mozilla.org/prod/usage_report_data/v1/master/fxhealth.json). 17 | It can also process Redash dashboards (see this [example configuration 18 | file](docs/example-redash-config.json)). Let us know if you have any questions 19 | or if you have a dataset that you would like us to spruce up. 20 | 21 | ## API 22 | 23 | Re-formatted data is currently hosted under the data.firefox.com domain, but you 24 | are also welcome to run ensemble-transposer yourself and host the re-formatted 25 | data elsewhere. 26 | 27 | * **Valid `platform` values:** *desktop* 28 | * **Valid `datasetName` values:** *hardware*, *user-activity*, *usage-behavior* 29 | * **Valid `categoryName` values:** Listed in the output of the 30 | */datasets/[platform]/[datasetName]* endpoint 31 | * **Valid `metricName` values:** Listed in the output of the 32 | */datasets/[platform]/[datasetName]* endpoint 33 | 34 | ### /datasets/[platform]/[datasetName]/index.json 35 | 36 | For example: https://data.firefox.com/datasets/desktop/user-activity/index.json 37 | 38 | A summary of the given dataset. For example, this includes a description of the 39 | dataset and a list of all metrics within it. 40 | 41 | ### /datasets/[platform]/[datasetName]/[categoryName]/[metricName]/index.json 42 | 43 | For example: https://data.firefox.com/datasets/desktop/user-activity/Italy/YAU/index.json 44 | 45 | Everything you need to know about a given metric in a given category. For 46 | example, this includes a title, a description, and a set of suggested axis 47 | labels. 48 | 49 | ## Development 50 | 51 | ### Setup 52 | 53 | 1. Install [Docker](https://docs.docker.com/install/) 54 | 2. Create a new [Amazon S3](https://aws.amazon.com/s3/) bucket 55 | 3. Copy *.env-dist* to *.env* and provide values for all environment variables 56 | 57 | ### Inspecting output 58 | 59 | Run `make start` and inspect that data that is uploaded to S3. 60 | 61 | ### Testing 62 | 63 | Run `make test` to lint code and run standard tests. 64 | 65 | Run `make compare` to compare the data in your S3 bucket to the data in the 66 | production S3 bucket. This can be useful when upgrading packages or refactoring 67 | code, for example. 68 | 69 | ## Deployment 70 | 71 | ### AWS 72 | 73 | This project was originally meant to be run as a cloud task, like a Lambda function or 74 | Google Cloud Function. The main function is specified as the value of `main` in 75 | *package.json*. Most services read this value and do the right thing. If not, 76 | you may need to manually point your service to that function. 77 | 78 | Before triggering the function, be sure to create an [Amazon 79 | S3](https://aws.amazon.com/s3/) bucket and set the following environment 80 | variables: 81 | 82 | * `AWS_BUCKET_NAME` 83 | * `AWS_REGION` 84 | * `AWS_ACCESS_KEY_ID` 85 | * `AWS_SECRET_ACCESS_KEY` 86 | 87 | ### Google Cloud 88 | 89 | This project can be run as a Docker container. The default command is `npm start`, but it may need 90 | to be explicitly configured in some environments. When running the container in GKE, authentication 91 | will be automatically detected. Before running, be sure to create a [Google Cloud 92 | Storage](https://cloud.google.com/storage) bucket and set the following environment variable: 93 | 94 | * `GCS_BUCKET_NAME` 95 | 96 | ### Other 97 | 98 | When neither `AWS_BUCKET_NAME` nor `GCS_BUCKET_NAME` are present in the environment, this project 99 | will write data to `./target`, which can then be copied to otherwise unsupported systems. 100 | 101 | ## Notes 102 | 103 | ### Versioning 104 | 105 | We maintain a version number for this project in *package.json*. It should be 106 | incremented whenever new code is pushed. 107 | 108 | The number looks like a semantic version number, but [semver isn't meant for 109 | applications](https://softwareengineering.stackexchange.com/a/255201). We 110 | instead follow these basic guidelines: the first number is incremented for major 111 | changes, the second number is incremented for medium-sized changes, and the 112 | third number is incremented for small changes. 113 | -------------------------------------------------------------------------------- /config/datasets/hardware.json: -------------------------------------------------------------------------------- 1 | { 2 | "sources": { 3 | "desktop": { 4 | "data": { 5 | "url": "https://analysis-output.telemetry.mozilla.org/public-data-report/hardware/hwsurvey-weekly.json", 6 | "format": "babbage" 7 | }, 8 | "annotations": { 9 | "url": "https://analysis-output.telemetry.mozilla.org/public-data-report/annotations/annotations_hardware.json" 10 | } 11 | } 12 | }, 13 | "options": { 14 | "title": "Hardware Across the Web", 15 | "description": "Hardware Across the Web is a public weekly report of the hardware used by a representative sample of the population from Firefox's release channel on desktop. This information can be used by developers to improve the Firefox experience for users.", 16 | "metaDescription": "The hardware of Firefox users, including graphics, processors, operating systems, and plugins. Use our public data to improve your Firefox support!", 17 | "metrics": { 18 | "resolution": { 19 | "title": "Display Resolution", 20 | "description": "Two display resolutions, 1920x1080px and 1366x768px, stand out as the most highly-used, with the former showing an upward trend.", 21 | "patterns": { 22 | "fields": "^resolution_", 23 | "populations": ".*?_(.*)" 24 | }, 25 | "type": "line", 26 | "axes": { 27 | "y": { 28 | "unit": "%" 29 | } 30 | } 31 | }, 32 | "gpuModel": { 33 | "title": "GPU Model", 34 | "description": [ 35 | "Share of primary GPU models", 36 | "Note: this report only includes the primary GPU for each machine. Any supplemental GPUs are not included and this report is not representative of accelerated graphics." 37 | ], 38 | "patterns": { 39 | "fields": "^gpuModel_", 40 | "populations": ".*?_(.*)" 41 | }, 42 | "populationModifications": { 43 | "renames": [ 44 | { 45 | "from": "gen7.5-haswell-gt2", 46 | "to": "Haswell (GT2)" 47 | }, 48 | { 49 | "from": "gen7-ivybridge-gt2", 50 | "to": "Ivy Bridge (GT2)" 51 | }, 52 | { 53 | "from": "gen6-sandybridge-gt2", 54 | "to": "Sandy Bridge (GT2)" 55 | }, 56 | { 57 | "from": "gen6-sandybridge-gt1", 58 | "to": "Sandy Bridge (GT1)" 59 | }, 60 | { 61 | "from": "gen7-ivybridge-gt1", 62 | "to": "Ivy Bridge (GT1)" 63 | }, 64 | { 65 | "from": "gen4.5-gma4500hd", 66 | "to": "GMA 4500HD" 67 | }, 68 | { 69 | "from": "gen7-baytrail", 70 | "to": "Bay Trail" 71 | }, 72 | { 73 | "from": "gen4.5-gma4500", 74 | "to": "GMA 4500" 75 | }, 76 | { 77 | "from": "gen8-broadwell-gt2", 78 | "to": "Broadwell (GT2)" 79 | }, 80 | { 81 | "from": "gen3-gma3100", 82 | "to": "GMA 3100" 83 | }, 84 | { 85 | "from": "gen3-gma950", 86 | "to": "GMA 950" 87 | }, 88 | { 89 | "from": "gen7.5-haswell-gt21", 90 | "to": "Haswell (GT21)" 91 | }, 92 | { 93 | "from": "gen7-ivybridge-gt22", 94 | "to": "Ivy Bridge (GT22)" 95 | }, 96 | { 97 | "from": "gen6-sandybridge-gt23", 98 | "to": "Sandy Bridge (GT23)" 99 | }, 100 | { 101 | "from": "gen6-sandybridge-gt14", 102 | "to": "Sandy Bridge (GT14)" 103 | }, 104 | { 105 | "from": "gen7-ivybridge-gt15", 106 | "to": "Ivy Bridge (GT15)" 107 | }, 108 | { 109 | "from": "gen4.5-gma4500hd6", 110 | "to": "GMA 4500HD 6" 111 | }, 112 | { 113 | "from": "gen7-baytrail8", 114 | "to": "Bay Trail 8" 115 | }, 116 | { 117 | "from": "gen4.5-gma45009", 118 | "to": "GMA 4500 9" 119 | }, 120 | { 121 | "from": "gen8-broadwell-gt210", 122 | "to": "Broadwell (GT 210)" 123 | }, 124 | { 125 | "from": "gen3-gma310011", 126 | "to": "GMA 310011" 127 | }, 128 | { 129 | "from": "EVERGREEN-PALM", 130 | "to": "Evergreen (Palm)" 131 | }, 132 | { 133 | "from": "gen9-skylake-gt2", 134 | "to": "Skylake (GT2)" 135 | }, 136 | { 137 | "from": "EVERGREEN-CEDAR", 138 | "to": "Evergreen (Cedar)" 139 | }, 140 | { 141 | "from": "CAYMAN-ARUBA", 142 | "to": "Cayman (Aruba)" 143 | }, 144 | { 145 | "from": "gen4-gma3500", 146 | "to": "GMA 3500" 147 | }, 148 | { 149 | "from": "Tesla-GT218", 150 | "to": "GeForce GT218" 151 | }, 152 | { 153 | "from": "NV40-C61", 154 | "to": "GeForce NV40" 155 | }, 156 | { 157 | "from": "gen7.5-haswell-gt3", 158 | "to": "Haswell (GT3)" 159 | }, 160 | { 161 | "from": "gen7.5-haswell-gt1", 162 | "to": "Haswell (GT1)" 163 | }, 164 | { 165 | "from": "EVERGREEN-TURKS", 166 | "to": "Evergreen (Turks)" 167 | }, 168 | { 169 | "from": "gen8-cherryview", 170 | "to": "Cherry View" 171 | }, 172 | { 173 | "from": "gen9-kabylake-gt2", 174 | "to": "Kaby Lake (GT2)" 175 | }, 176 | { 177 | "from": "gen8-broadwell-gt3", 178 | "to": "Broadwell (GT3)" 179 | }, 180 | { 181 | "from": "gen5-ironlake", 182 | "to": "Ironlake" 183 | }, 184 | { 185 | "from": "gen9-coffeelake-gt2", 186 | "to": "Coffee Lake (GT2)" 187 | }, 188 | { 189 | "from": "gen9-kabylake-gt1.5", 190 | "to": "Kaby Lake (GT1.5)" 191 | } 192 | ], 193 | "exclusions": [ 194 | "Other" 195 | ] 196 | }, 197 | "type": "line", 198 | "axes": { 199 | "y": { 200 | "unit": "%" 201 | } 202 | } 203 | }, 204 | "gpuVendor": { 205 | "title": "GPU Vendor", 206 | "description": [ 207 | "Intel makes up the largest part of the desktop GPU market, accounting for over 65% of our release Desktop population. AMD and Nvidia come in virtually tied for 2^nd^.", 208 | "Note: this report only includes the primary GPU for each machine. Any supplemental GPUs are not included and this report is not representative of accelerated graphics." 209 | ], 210 | "patterns": { 211 | "fields": "^gpuVendor_", 212 | "populations": ".*?_(.*)" 213 | }, 214 | "type": "line", 215 | "axes": { 216 | "y": { 217 | "unit": "%" 218 | } 219 | } 220 | }, 221 | "cpuSpeed": { 222 | "title": "CPU Speeds", 223 | "description": "About 20% of users have processors with clock speeds between 2.3 GHz and 2.69 GHz.", 224 | "patterns": { 225 | "fields": "^cpuSpeed", 226 | "populations": ".*?_(.*)" 227 | }, 228 | "populationModifications": { 229 | "replacementGroups": [ 230 | { 231 | "name": "Less than 1.4 GHz", 232 | "memberPattern": "^(0\\.[0-9]+|1\\.[0-3][0-9]*)$" 233 | }, 234 | { 235 | "name": "1.4 GHz to 1.49 GHz", 236 | "memberPattern": "^1\\.4[0-9]*$" 237 | }, 238 | { 239 | "name": "1.5 GHz to 1.69 GHz", 240 | "memberPattern": "^1\\.(5[0-9]*|6[0-6]*[0-9]*)$" 241 | }, 242 | { 243 | "name": "1.7 GHz to 1.99 GHz", 244 | "memberPattern": "^1\\.[7-9][0-9]*$" 245 | }, 246 | { 247 | "name": "2.0 GHz to 2.29 GHz", 248 | "memberPattern": "^2\\.[0-2][0-9]*$" 249 | }, 250 | { 251 | "name": "2.3 GHz to 2.69 GHz", 252 | "memberPattern": "^2\\.[3-6][0-9]*$" 253 | }, 254 | { 255 | "name": "2.7 GHz to 2.99 GHz", 256 | "memberPattern": "^2\\.[7-9][0-9]*$" 257 | }, 258 | { 259 | "name": "3.0 GHz to 3.29 GHz", 260 | "memberPattern": "^3\\.[0-2][0-9]*$" 261 | }, 262 | { 263 | "name": "3.3 GHz to 3.69 GHz", 264 | "memberPattern": "^3\\.[3-6][0-9]*$" 265 | }, 266 | { 267 | "name": "3.7 GHz to 3.99 GHz", 268 | "memberPattern": "^3\\.[7-9][0-9]*$" 269 | }, 270 | { 271 | "name": "More than 4.0 GHz", 272 | "memberPattern": "^([4-9]|[0-9][0-9]+)\\.[0-9]+$" 273 | } 274 | ] 275 | }, 276 | "type": "line", 277 | "axes": { 278 | "y": { 279 | "unit": "%" 280 | } 281 | } 282 | }, 283 | "ram": { 284 | "title": "Memory", 285 | "description": "The most popular memory sizes are 8GB and 16GB, with almost one-third and one-fifth of our Release users, respectively.", 286 | "patterns": { 287 | "fields": "^ram_", 288 | "populations": ".*?_(.*)" 289 | }, 290 | "populationModifications": { 291 | "append": { 292 | "matchPattern": "\\d+", 293 | "value": "GB" 294 | } 295 | }, 296 | "type": "line", 297 | "axes": { 298 | "y": { 299 | "unit": "%" 300 | } 301 | } 302 | }, 303 | "cpuCores": { 304 | "title": "CPU Cores", 305 | "description": "Nearly 34% of users have machines with two physical cores, and an additional 34% of users have machines with four physical cores.", 306 | "patterns": { 307 | "fields": "^cpuCores_", 308 | "populations": ".*?_(.*)" 309 | }, 310 | "type": "line", 311 | "axes": { 312 | "y": { 313 | "unit": "%" 314 | } 315 | } 316 | }, 317 | "cpuVendor": { 318 | "title": "CPU Vendor", 319 | "description": "Intel leads the share of CPUs found in our Release users, with about 82% of profiles, while AMD follows behind with about 14%.", 320 | "patterns": { 321 | "fields": "^cpuVendor", 322 | "populations": ".*?_(.*)" 323 | }, 324 | "populationModifications": { 325 | "renames": [ 326 | { 327 | "from": "GenuineIntel", 328 | "to": "Intel" 329 | }, 330 | { 331 | "from": "AuthenticAMD", 332 | "to": "AMD" 333 | } 334 | ] 335 | }, 336 | "type": "line", 337 | "axes": { 338 | "y": { 339 | "unit": "%" 340 | } 341 | } 342 | }, 343 | "osName": { 344 | "title": "Operating System", 345 | "description": "Windows 10 and Windows 11 are the most popular operating systems, accounting for around 75% of our users.", 346 | "patterns": { 347 | "fields": "^osName_", 348 | "populations": ".*?_(.*)" 349 | }, 350 | "populationModifications": { 351 | "renames": [ 352 | { 353 | "from": "Windows_NT-5.1", 354 | "to": "Windows XP" 355 | }, 356 | { 357 | "from": "Windows_NT-6.0", 358 | "to": "Windows Vista" 359 | }, 360 | { 361 | "from": "Windows_NT-6.1", 362 | "to": "Windows 7" 363 | }, 364 | { 365 | "from": "Windows_NT-6.2", 366 | "to": "Windows 8" 367 | }, 368 | { 369 | "from": "Windows_NT-6.3", 370 | "to": "Windows 8.1" 371 | }, 372 | { 373 | "from": "Windows_NT-10.0", 374 | "to": "Windows 10" 375 | }, 376 | { 377 | "from": "Windows_NT-10.0.2xxxx", 378 | "to": "Windows 11" 379 | }, 380 | { 381 | "from": "Windows_NT-Other", 382 | "to": "Windows Other" 383 | }, 384 | { 385 | "from": "Linux-Other", 386 | "to": "Linux Other" 387 | }, 388 | { 389 | "from": "Darwin-14.x", 390 | "to": "macOS Yosemite" 391 | }, 392 | { 393 | "from": "Darwin-15.x", 394 | "to": "macOS El Capitan" 395 | }, 396 | { 397 | "from": "Darwin-16.x", 398 | "to": "macOS Sierra" 399 | }, 400 | { 401 | "from": "Darwin-17.x", 402 | "to": "macOS High Sierra" 403 | }, 404 | { 405 | "from": "Darwin-18.x", 406 | "to": "macOS Mojave" 407 | }, 408 | { 409 | "from": "Darwin-19.x", 410 | "to": "macOS Catalina" 411 | }, 412 | { 413 | "from": "Darwin-20.x", 414 | "to": "macOS Big Sur" 415 | }, 416 | { 417 | "from": "Darwin-Other", 418 | "to": "macOS Other" 419 | } 420 | ] 421 | }, 422 | "type": "line", 423 | "axes": { 424 | "y": { 425 | "unit": "%" 426 | } 427 | } 428 | }, 429 | "browserArch": { 430 | "title": "Browsers by Architecture", 431 | "description": "In 2017, 64-bit Firefox updates fully unthrottled on Win7+ for 2GB+ users.", 432 | "patterns": { 433 | "fields": "^browserArch_", 434 | "populations": ".*?_(.*)" 435 | }, 436 | "populationModifications": { 437 | "renames": [ 438 | { 439 | "from": "x86", 440 | "to": "32-bit" 441 | }, 442 | { 443 | "from": "x86-64", 444 | "to": "64-bit" 445 | }, 446 | { 447 | "from": "aarch64", 448 | "to": "64-bit ARM" 449 | } 450 | ] 451 | }, 452 | "type": "line", 453 | "axes": { 454 | "y": { 455 | "unit": "%" 456 | } 457 | } 458 | }, 459 | "osArch": { 460 | "title": "Operating Systems by Architecture", 461 | "description": "We see 64-bit systems accounting for over 85% of our users. Note that this metric likely undercounts non-Windows OSs, as it relies on the environment.system.isWow64 Telemetry field.", 462 | "patterns": { 463 | "fields": "^osArch_", 464 | "populations": ".*?_(.*)" 465 | }, 466 | "populationModifications": { 467 | "renames": [ 468 | { 469 | "from": "x86", 470 | "to": "32-bit" 471 | }, 472 | { 473 | "from": "x86-64", 474 | "to": "64-bit" 475 | }, 476 | { 477 | "from": "aarch64", 478 | "to": "64-bit ARM" 479 | } 480 | ] 481 | }, 482 | "type": "line", 483 | "axes": { 484 | "y": { 485 | "unit": "%" 486 | } 487 | } 488 | }, 489 | "hasFlash": { 490 | "title": "Has Flash", 491 | "description": "Since Adobe stopped supporting Flash Player beginning December 31, 2020, Flash availability on Firefox is now below 1%.", 492 | "patterns": { 493 | "fields": "^hasFlash_True$" 494 | }, 495 | "type": "line", 496 | "axes": { 497 | "y": { 498 | "unit": "%" 499 | } 500 | } 501 | } 502 | }, 503 | "summaryMetrics": [ 504 | "gpuVendor", 505 | "cpuVendor", 506 | "osName", 507 | "hasFlash" 508 | ], 509 | "dashboard": { 510 | "sectioned": true, 511 | "sections": [ 512 | { 513 | "key": "graphics", 514 | "title": "Graphics", 515 | "metrics": ["gpuModel", "gpuVendor", "resolution"] 516 | }, 517 | { 518 | "key": "processor", 519 | "title": "Processor", 520 | "metrics": ["cpuVendor", "cpuCores", "cpuSpeed", "ram"] 521 | }, 522 | { 523 | "key": "operating-system", 524 | "title": "Operating System", 525 | "metrics": ["osName", "browserArch", "osArch"] 526 | }, 527 | { 528 | "key": "plugins", 529 | "title": "Plugins", 530 | "metrics": ["hasFlash"] 531 | } 532 | ] 533 | } 534 | } 535 | } 536 | -------------------------------------------------------------------------------- /config/datasets/usage-behavior.json: -------------------------------------------------------------------------------- 1 | { 2 | "sources": { 3 | "desktop": { 4 | "data": { 5 | "url": "https://analysis-output.telemetry.mozilla.org/public-data-report/user_activity/webusage.json", 6 | "format": "quantum" 7 | }, 8 | "annotations": { 9 | "url": "https://analysis-output.telemetry.mozilla.org/public-data-report/annotations/annotations_webusage.json" 10 | } 11 | } 12 | }, 13 | "options": { 14 | "title": "Usage Behavior", 15 | "description": "Usage Behavior is a weekly report describing ways in which desktop users are interacting with the web.", 16 | "metaDescription": "How Firefox users interact with the web, including top languages, tracking protection, and top add-ons. Use our public data to understand your audience!", 17 | "defaultCategory": "Worldwide", 18 | "metrics": { 19 | "locale": { 20 | "title": "Top Languages", 21 | "description": [ 22 | "Top Languages shows the distribution of the top 5 language settings for Firefox Desktop.", 23 | "Worldwide, English (US) remains the most common, at about 40% of the population, with German (11%) and French (8.1%) coming 2^nd^ and 3^rd^. Simplified Chinese is the 4^th^ most common language (6.7%), and Spanish (Spain) is the 5^th^ most common language (5%).", 24 | "For most countries in the top 10, the majority (>90%) of users have their language set to the local language, with a notable exception in Indonesia, which has about 71% English (US) and 27% Indonesian." 25 | ], 26 | "type": "line", 27 | "axes": { 28 | "y": { 29 | "unit": "%" 30 | } 31 | } 32 | }, 33 | "pct_addon": { 34 | "title": "Has Add-on", 35 | "description": [ 36 | "Has Add-on shows the percentage of Firefox Desktop clients with user-installed add-ons.", 37 | "One of the best things about Firefox is the robust add-on community which gives users the option to customize and control their browsing experience. Our users agree, with over 40% of Firefox users having at least 1 installed add-on.", 38 | "Add-on usage measured here reflects multiple facets of browser customization, including web extensions, language packs, and themes.", 39 | "This metric varies globally. On one end is India, where 19% of users have add-ons, and on the other is Russia and Canada, where almost 60% of users have add-ons." 40 | ], 41 | "type": "line", 42 | "axes": { 43 | "y": { 44 | "unit": "%" 45 | } 46 | } 47 | }, 48 | "top10addons": { 49 | "title": "Top Add-ons", 50 | "description": [ 51 | "Top Add-ons shows the top 10 most popular Firefox Desktop add-ons for a week.", 52 | "Overall, the most popular add-ons are ad-blockers, coming in first in almost all of the top 10 countries. Ad-blocking seems particularly popular in Germany and France, with 4 of the top 10 add-ons being ad-blockers in France." 53 | ], 54 | "type": "table", 55 | "columns": [ 56 | { 57 | "name": "Add-on" 58 | }, 59 | { 60 | "name": "Usage", 61 | "unit": "%" 62 | } 63 | ] 64 | } 65 | }, 66 | "dashboard": { 67 | "sectioned": false, 68 | "metrics": [ 69 | "locale", 70 | "pct_TP", 71 | "pct_addon", 72 | "top10addons" 73 | ] 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /config/datasets/user-activity.json: -------------------------------------------------------------------------------- 1 | { 2 | "sources": { 3 | "desktop": { 4 | "data": { 5 | "url": "https://analysis-output.telemetry.mozilla.org/public-data-report/user_activity/fxhealth.json", 6 | "format": "quantum" 7 | }, 8 | "annotations": { 9 | "url": "https://analysis-output.telemetry.mozilla.org/public-data-report/annotations/annotations_fxhealth.json" 10 | } 11 | } 12 | }, 13 | "options": { 14 | "title": "User Activity", 15 | "description": "Usage Behavior is a weekly report describing ways in which users are interacting with the web.", 16 | "metaDescription": "The state of the Firefox userbase, including user totals, usage hours, and version adoption. Use our public data to improve your Firefox support!", 17 | "defaultCategory": "Worldwide", 18 | "metrics": { 19 | "MAU": { 20 | "title": "Monthly Active Users", 21 | "description": [ 22 | "Monthly Active Users (MAU) measures the number of Desktop and Mobile clients active in the past 28 days. MAU typically fluctuates over the year: dipping in the summer, around the New Year, and during major holidays." 23 | ], 24 | "type": "line", 25 | "axes": { 26 | "y": { 27 | "unit": "clients" 28 | } 29 | } 30 | }, 31 | "avg_daily_usage(hours)": { 32 | "title": "Daily Usage", 33 | "description": [ 34 | "Daily Usage shows the hours spent browsing for a typical Firefox Desktop client in a typical day of use. Globally, the typical Firefox client averages around 4.5 hours of use per day." 35 | ], 36 | "type": "line", 37 | "axes": { 38 | "y": { 39 | "unit": "hours per day" 40 | } 41 | } 42 | }, 43 | "avg_intensity": { 44 | "title": "Average Intensity", 45 | "description": [ 46 | "Intensity shows how many days per week users use Firefox Desktop. Overall, the typical Firefox client uses the browser 3.5 days per week." 47 | ], 48 | "type": "line", 49 | "axes": { 50 | "y": { 51 | "unit": "intensity" 52 | } 53 | } 54 | }, 55 | "pct_new_user": { 56 | "title": "New Profile Rate", 57 | "description": [ 58 | "New Profile Rate measures how often new Firefox Desktop profiles are created. A profile is the folder Firefox uses to store your add-ons, settings, and other customizations; essentially, your browser's identity.", 59 | "We measure profiles rather than “new users” because Firefox doesn't track individuals. Profiles let us understand how the browser is used without identifying who's using it. It makes our job harder, but we think your privacy is worth it." 60 | ], 61 | "type": "line" 62 | }, 63 | "pct_latest_version": { 64 | "title": "Latest Version", 65 | "description": [ 66 | "Latest Version shows the percentage of Firefox Desktop clients running the latest version of Firefox or greater (for that week). Firefox typically releases major updates roughly every 30 days and generally releases on Tuesdays." 67 | ], 68 | "type": "line", 69 | "axes": { 70 | "y": { 71 | "unit": "%" 72 | } 73 | } 74 | } 75 | }, 76 | "dashboard": { 77 | "sectioned": false, 78 | "metrics": [ 79 | "YAU", 80 | "MAU", 81 | "avg_daily_usage(hours)", 82 | "avg_intensity", 83 | "pct_new_user", 84 | "pct_latest_version" 85 | ] 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /docs/example-redash-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "sources": { 3 | "data": { 4 | "url": { 5 | "base": "https://sql.telemetry.mozilla.org/api/dashboards/[dashboard-key]", 6 | "query": { 7 | "api_key": "REDASH_API_KEY" 8 | } 9 | }, 10 | "format": "redash" 11 | } 12 | }, 13 | "options": { 14 | "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat." 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ensemble-transposer", 3 | "version": "3.5.0", 4 | "private": true, 5 | "description": "ensemble-transposer re-formats existing data so that it can be used by the Firefox Public Data Report.", 6 | "repository": { 7 | "type": "git", 8 | "url": "git+https://github.com/mozilla/ensemble-transposer.git" 9 | }, 10 | "license": "MPL-2.0", 11 | "bugs": { 12 | "url": "https://github.com/mozilla/ensemble-transposer/issues" 13 | }, 14 | "homepage": "https://github.com/mozilla/ensemble-transposer#readme", 15 | "engines": { 16 | "node": ">=8.16.2" 17 | }, 18 | "main": "src/index.js", 19 | "scripts": { 20 | "start": "node --eval \"require('./src').default();\"", 21 | "test": "npm run lint && npm run mocha", 22 | "lint": "eslint --ext=.js,.json .", 23 | "premocha": "npm start", 24 | "mocha": "mocha src/tests/standard", 25 | "precompare": "npm start", 26 | "compare": "mocha src/tests/special/api-equivalence.test.js", 27 | "posttest": "npm audit || true" 28 | }, 29 | "dependencies": { 30 | "@google-cloud/storage": "^6.9.5", 31 | "aws-sdk": "^2.1692.0", 32 | "big.js": "5.2.2", 33 | "dotenv": "8.2.0", 34 | "fs-extra": "^11.1.1", 35 | "memoizee": "0.4.14", 36 | "node-fetch": "^2.7.0" 37 | }, 38 | "devDependencies": { 39 | "chai": "4.2.0", 40 | "deep-equal-in-any-order": "1.0.28", 41 | "eslint": "7.32.0", 42 | "eslint-plugin-json": "2.1.2", 43 | "eslint-plugin-mocha": "7.0.1", 44 | "eslint-plugin-node": "11.1.0", 45 | "mocha": "8.4.0" 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/formatters/BabbageFormatter.js: -------------------------------------------------------------------------------- 1 | const Big = require('big.js'); 2 | 3 | const Formatter = require('./Formatter'); 4 | 5 | /** 6 | * Format data which is structured like the old Firefox Hardware Report data. 7 | * 8 | * This was previously done by a project named workshop a.k.a. fhwr-unflattener. 9 | * https://github.com/mozilla/workshop 10 | * 11 | * Although we can format this type of data, we may not want to advertise this 12 | * fact. The configuration file is much more complex. The Quantum format is much 13 | * easier to work with. 14 | */ 15 | module.exports = class extends Formatter { 16 | async getSummary() { 17 | this.apiVersion = '2.0.0'; 18 | this.defaultCategory = 'default'; 19 | this.defaultPopulation = 'default'; 20 | this.valueMultiplier = 100; 21 | 22 | const summary = {}; 23 | 24 | const unsortedDates = new Set(this.rawData.map(entry => entry.date)); 25 | const sortedDates = Array.from(unsortedDates).sort().reverse(); 26 | 27 | summary.title = this.config.options.title; 28 | summary.description = this.config.options.description; 29 | 30 | if (this.config.options.metaDescription) { 31 | summary.metaDescription = this.config.options.metaDescription; 32 | } 33 | 34 | summary.categories = [this.defaultCategory]; 35 | summary.metrics = Object.keys(this.config.options.metrics); 36 | 37 | if (this.config.options.summaryMetrics) { 38 | summary.summaryMetrics = this.config.options.summaryMetrics; 39 | } 40 | 41 | summary.dates = sortedDates; 42 | 43 | if (this.config.options.dashboard && this.config.options.dashboard.sectioned) { 44 | summary.sections = this.config.options.dashboard.sections; 45 | } 46 | 47 | summary.apiVersion = this.apiVersion; 48 | 49 | return summary; 50 | } 51 | 52 | async getMetric(categoryName, metricName) { 53 | const metric = {}; 54 | 55 | const metricConfig = this.config.options.metrics[metricName]; 56 | const data = { populations: {} }; 57 | 58 | const fieldsRegex = new RegExp(metricConfig.patterns.fields); 59 | this.rawData.forEach(entry => { 60 | const matchingFields = Object.keys(entry).filter(fieldName => { 61 | return fieldsRegex.test(fieldName); 62 | }); 63 | 64 | if (matchingFields.length === 1) { 65 | data.populations[this.defaultPopulation] = data.populations[this.defaultPopulation] || []; 66 | this.pushSingleDataPoint( 67 | data.populations[this.defaultPopulation], 68 | entry, 69 | matchingFields[0], 70 | ) 71 | } else if (matchingFields.length > 1) { 72 | if (!metricConfig.patterns.populations) { 73 | throw new Error(`No population pattern specified for metric "${metricName}" in dataset "${this.datasetName}"`); 74 | } 75 | 76 | this.pushMultiplePopulations( 77 | data.populations, 78 | metricConfig, 79 | entry, 80 | matchingFields, 81 | metricConfig.patterns.populations, 82 | ) 83 | } 84 | }); 85 | 86 | const annotations = this.getAnnotations(categoryName, metricName); 87 | 88 | metric.title = metricConfig.title; 89 | metric.description = metricConfig.description; 90 | metric.type = metricConfig.type; 91 | 92 | if (metricConfig.axes) { 93 | metric.axes = metricConfig.axes; 94 | } 95 | 96 | metric.data = data; 97 | 98 | if (annotations) { 99 | metric.annotations = annotations; 100 | } 101 | 102 | metric.apiVersion = this.apiVersion; 103 | 104 | return metric; 105 | } 106 | 107 | pushSingleDataPoint(arr, entry, field) { 108 | const value = parseFloat(new Big( 109 | entry[field] 110 | ).times(this.valueMultiplier)); 111 | 112 | arr.push({ 113 | x: entry.date, 114 | y: value, 115 | }); 116 | } 117 | 118 | pushMultiplePopulations(obj, metricConfig, entry, matchingFields, populationsPattern) { 119 | let createdAnyGroups = false; 120 | let groupTotals = {}; 121 | 122 | matchingFields.forEach(fieldName => { 123 | const fieldValue = entry[fieldName]; 124 | const populationsRegex = new RegExp(populationsPattern); 125 | 126 | // This isn't really documented anywhere (except here I guess) but 127 | // the populations regex should always contain exactly one group 128 | // which represents the population name. 129 | // 130 | // In other words, we don't care about the whole match. Just the 131 | // first group. 132 | const rawPopulationName = fieldName.match(populationsRegex)[1]; 133 | 134 | const populationName = this.getPopulationName( 135 | metricConfig, 136 | rawPopulationName, 137 | ); 138 | 139 | const replacementGroup = this.getReplacementGroup(metricConfig, rawPopulationName); 140 | 141 | if (replacementGroup) { 142 | createdAnyGroups = true; 143 | 144 | if (groupTotals[replacementGroup.name]) { 145 | groupTotals[replacementGroup.name] = parseFloat(new Big( 146 | groupTotals[replacementGroup.name] 147 | ).plus(fieldValue)); 148 | } else { 149 | groupTotals[replacementGroup.name] = fieldValue; 150 | } 151 | } else if (!this.populationIsExcluded(metricConfig, rawPopulationName)) { 152 | const value = parseFloat(new Big( 153 | fieldValue 154 | ).times(this.valueMultiplier)); 155 | 156 | obj[populationName] = obj[populationName] || []; 157 | obj[populationName].push({ 158 | x: entry.date, 159 | y: value, 160 | }); 161 | } 162 | }); 163 | 164 | if (createdAnyGroups) { 165 | Object.keys(groupTotals).forEach(groupName => { 166 | const value = parseFloat(new Big( 167 | groupTotals[groupName] 168 | ).times(this.valueMultiplier)); 169 | 170 | obj[groupName] = obj[groupName] || []; 171 | obj[groupName].push({ 172 | x: entry.date, 173 | y: value, 174 | }); 175 | }); 176 | } 177 | } 178 | 179 | getPopulationName(metricConfig, rawPopulationName) { 180 | if (!metricConfig.populationModifications) return rawPopulationName; 181 | 182 | let populationName = rawPopulationName; 183 | 184 | if (metricConfig.populationModifications.renames) { 185 | const rename = metricConfig.populationModifications.renames.find(r => { 186 | return r.from === rawPopulationName; 187 | }); 188 | 189 | if (rename) { 190 | populationName = rename.to; 191 | } 192 | } else if (metricConfig.populationModifications.append) { 193 | const append = metricConfig.populationModifications.append; 194 | const appendRegex = new RegExp(append.matchPattern); 195 | if (appendRegex.test(rawPopulationName)) { 196 | populationName = rawPopulationName + append.value; 197 | } 198 | } 199 | 200 | return populationName; 201 | } 202 | 203 | getReplacementGroup(metricConfig, rawPopulationName) { 204 | if (!metricConfig.populationModifications) return; 205 | if (!metricConfig.populationModifications.replacementGroups) return; 206 | 207 | return metricConfig.populationModifications.replacementGroups.find(rg => { 208 | const memberRegex = new RegExp(rg.memberPattern); 209 | return memberRegex.test(rawPopulationName); 210 | }); 211 | } 212 | 213 | populationIsExcluded(metricConfig, rawPopulationName) { 214 | if (!metricConfig.populationModifications) return false; 215 | if (!metricConfig.populationModifications.exclusions) return false; 216 | 217 | return metricConfig.populationModifications.exclusions.includes( 218 | rawPopulationName 219 | ); 220 | } 221 | } 222 | -------------------------------------------------------------------------------- /src/formatters/Formatter.js: -------------------------------------------------------------------------------- 1 | module.exports = class { 2 | constructor(datasetName, platform, config, rawData, rawAnnotations) { 3 | this.datasetName = datasetName; 4 | this.platform = platform; 5 | this.config = config; 6 | this.rawData = rawData; 7 | this.rawAnnotations = rawAnnotations; 8 | 9 | this.checkForErrors(); 10 | } 11 | 12 | getSummary() { 13 | throw new Error(`getSummary not implemented for format ${this.config.sources[this.platform].data.format}`); 14 | } 15 | 16 | getMetric() { 17 | throw new Error(`getMetric not implemented for format ${this.config.sources[this.platform].data.format}`); 18 | } 19 | 20 | getAnnotations(categoryName, metricName) { 21 | const annotations = []; 22 | 23 | if (this.rawAnnotations) { 24 | this.rawAnnotations[categoryName].forEach(annotation => { 25 | if (metricName in annotation.annotation) { 26 | annotations.push({ 27 | date: annotation.date, 28 | label: annotation.annotation[metricName], 29 | }); 30 | } 31 | }); 32 | } 33 | 34 | if (annotations.length) { 35 | return annotations; 36 | } 37 | } 38 | 39 | clearCache() { 40 | return; 41 | } 42 | 43 | checkForErrors() { 44 | return; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/formatters/QuantumFormatter.js: -------------------------------------------------------------------------------- 1 | const Formatter = require('./Formatter'); 2 | 3 | 4 | module.exports = class extends Formatter { 5 | constructor(...args) { 6 | super(...args); 7 | this.apiVersion = '2.0.0'; 8 | } 9 | 10 | async getSummary() { 11 | const summary = {}; 12 | 13 | const unsortedDates = new Set(Object.keys(this.rawData).reduce((acc, categoryName) => { 14 | return acc.concat(this.rawData[categoryName].map(e => e.date)); 15 | }, [])); 16 | const sortedDates = Array.from(unsortedDates).sort().reverse(); 17 | 18 | summary.title = this.config.options.title; 19 | summary.description = this.config.options.description; 20 | 21 | if (this.config.options.metaDescription) { 22 | summary.metaDescription = this.config.options.metaDescription; 23 | } 24 | 25 | summary.categories = Object.keys(this.rawData); 26 | 27 | if (this.config.options.defaultCategory) { 28 | summary.defaultCategory = this.config.options.defaultCategory; 29 | } 30 | 31 | summary.metrics = Object.keys(this.config.options.metrics); 32 | 33 | if (this.config.options.summaryMetrics) { 34 | summary.summaryMetrics = this.config.options.summaryMetrics; 35 | } 36 | 37 | summary.dates = sortedDates; 38 | 39 | if (this.config.options.dashboard && this.config.options.dashboard.sectioned) { 40 | summary.sections = this.config.options.dashboard.sections; 41 | } 42 | 43 | summary.apiVersion = this.apiVersion; 44 | 45 | return summary; 46 | } 47 | 48 | async getMetric(categoryName, metricName) { 49 | const metricConfig = this.config.options.metrics[metricName]; 50 | const metric = {}; 51 | 52 | let data; 53 | switch (metricConfig.type) { 54 | case 'line': 55 | data = this.formatLineData(categoryName, metricName); 56 | break; 57 | case 'table': 58 | data = this.formatTableData(categoryName, metricName); 59 | break; 60 | default: 61 | throw new Error(`Unsupported type "${metricConfig.type}" for metric "${metricName}" in dataset "${this.datasetName}"`); 62 | } 63 | 64 | const annotations = this.getAnnotations(categoryName, metricName); 65 | 66 | metric.title = metricConfig.title; 67 | metric.description = metricConfig.description; 68 | metric.type = metricConfig.type; 69 | 70 | if (metricConfig.type === 'line' && metricConfig.axes) { 71 | metric.axes = metricConfig.axes; 72 | } 73 | 74 | if (metricConfig.type === 'table') { 75 | metric.columns = metricConfig.columns; 76 | } 77 | 78 | metric.data = data; 79 | 80 | if (annotations) { 81 | metric.annotations = annotations; 82 | } 83 | 84 | metric.apiVersion = this.apiVersion; 85 | 86 | return metric; 87 | } 88 | 89 | formatLineData(categoryName, metricName) { 90 | const data = { populations: {} }; 91 | 92 | this.rawData[categoryName].forEach(entry => { 93 | const metricValue = entry.metrics[metricName]; 94 | 95 | // If this metric has no value at this date, move on... 96 | if (!metricValue) return; 97 | 98 | // Multiple populations 99 | if (typeof metricValue === 'object') { 100 | Object.keys(metricValue).forEach(populationName => { 101 | data.populations[populationName] = data.populations[populationName] || []; 102 | data.populations[populationName].push({ 103 | x: entry.date, 104 | y: metricValue[populationName], 105 | }); 106 | }); 107 | } 108 | 109 | // Single population 110 | else if (typeof metricValue === 'number') { 111 | data.populations.default = data.populations.default || []; 112 | data.populations.default.push({ 113 | x: entry.date, 114 | y: metricValue, 115 | }); 116 | } 117 | 118 | else { 119 | throw new Error(`Raw data is not formatted properly for metric "${metricName}" in dataset "${this.datasetName}"`); 120 | } 121 | }); 122 | 123 | return data; 124 | } 125 | 126 | formatTableData(categoryName, metricName) { 127 | const data = { dates: {} }; 128 | 129 | this.rawData[categoryName].forEach(entry => { 130 | Object.keys(entry.metrics[metricName]).forEach(rowName => { 131 | const rowValue = entry.metrics[metricName][rowName]; 132 | 133 | data.dates[entry.date] = data.dates[entry.date] || {}; 134 | data.dates[entry.date].rows = data.dates[entry.date].rows || []; 135 | 136 | data.dates[entry.date].rows.push({ 137 | name: rowName, 138 | value: rowValue, 139 | }); 140 | }); 141 | }); 142 | 143 | return data; 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /src/formatters/RedashFormatter.js: -------------------------------------------------------------------------------- 1 | const fetch = require("node-fetch"); 2 | const Formatter = require('./Formatter'); 3 | const memoize = require('memoizee'); 4 | 5 | 6 | module.exports = class extends Formatter { 7 | constructor(...args) { 8 | super(...args); 9 | this.apiVersion = '2.0.0'; 10 | 11 | this.getRawMetric = memoize(async metricName => { 12 | const visualization = this.getVisualization(metricName); 13 | const apiKey = visualization.query.api_key; 14 | const endpoint = `https://sql.telemetry.mozilla.org/api/queries/${metricName}/results.json?api_key=${apiKey}`; 15 | const response = await fetch(endpoint); 16 | return await response.json() 17 | }); 18 | } 19 | 20 | async getSummary() { 21 | const summary = {}; 22 | 23 | summary.title = this.rawData.name; 24 | summary.description = this.config.options.description; 25 | 26 | if (this.config.options.metaDescription) { 27 | summary.metaDescription = this.config.options.metaDescription; 28 | } 29 | 30 | summary.categories = ['default']; 31 | summary.metrics = this.rawData.widgets.map(w => w.visualization.query.id.toString()); 32 | 33 | if (this.config.options.summaryMetrics) { 34 | summary.summaryMetrics = this.config.options.summaryMetrics; 35 | } 36 | 37 | summary.dates = await this.getDates(summary.metrics); 38 | 39 | if (this.config.options.dashboard && this.config.options.dashboard.sectioned) { 40 | summary.sections = this.config.options.dashboard.sections; 41 | } 42 | 43 | summary.apiVersion = this.apiVersion; 44 | 45 | return summary; 46 | } 47 | 48 | async getMetric(categoryName, metricName) { 49 | const metric = {}; 50 | 51 | const rawMetric = await this.getRawMetric(metricName); 52 | const visualization = this.getVisualization(metricName); 53 | const rawDescription = visualization.query.description; 54 | const annotations = this.getAnnotations(categoryName, metricName); 55 | const axes = this.getAxes(visualization); 56 | 57 | metric.title = visualization.query.name; 58 | 59 | if (rawDescription) { 60 | metric.description = rawDescription.split(/[\r\n]+/).map(s => s.trim()); 61 | } 62 | 63 | metric.type = 'line'; 64 | 65 | if (axes) { 66 | metric.axes = axes; 67 | } 68 | 69 | metric.data = this.getData(metricName, rawMetric, visualization); 70 | 71 | if (annotations) { 72 | metric.annotations = annotations; 73 | } 74 | 75 | metric.apiVersion = this.apiVersion; 76 | 77 | return metric; 78 | } 79 | 80 | clearCache() { 81 | this.getRawMetric.clear(); 82 | } 83 | 84 | checkForErrors() { 85 | const visualizations = this.rawData.widgets.map(w => w.visualization); 86 | 87 | for (const visualization of visualizations) { 88 | const type = visualization.type; 89 | if (type !== 'CHART') { 90 | throw new Error(`Visualization type "${type}" is not supported in dataset "${this.datasetName}"`); 91 | } 92 | 93 | const gst = visualization.options.globalSeriesType; 94 | if (gst !== 'line') { 95 | throw new Error(`globalSeriesType "${gst}" is not supported in dataset "${this.datasetName}"`); 96 | } 97 | } 98 | } 99 | 100 | getVisualization(metricName) { 101 | return this.rawData.widgets.find(w => { 102 | return w.visualization.query.id === Number(metricName); 103 | }).visualization; 104 | } 105 | 106 | async getDates(metrics) { 107 | const dates = new Set(); 108 | 109 | for (const metricName of metrics) { 110 | const rawMetric = await this.getRawMetric(metricName); 111 | 112 | const xFieldName = this.getXFieldName(metricName); 113 | 114 | for (const row of rawMetric.query_result.data.rows) { 115 | dates.add(row[xFieldName].substring(0, 10)); 116 | } 117 | } 118 | 119 | return Array.from(dates).sort().reverse(); 120 | } 121 | 122 | // Ignore the x axis label for now. Ensemble currently only supports 123 | // time-series charts and it doesn't label their x axes. 124 | getAxes(visualization) { 125 | const axes = {}; 126 | const options = visualization.options; 127 | 128 | if (options.yAxis) { 129 | // There may be multiple y axes 130 | const firstLabelConfig = options.yAxis.find(y => { 131 | return y.title && y.title.text; 132 | }); 133 | 134 | if (firstLabelConfig) { 135 | axes.y = { 136 | unit: firstLabelConfig.title.text, 137 | }; 138 | } 139 | } 140 | 141 | if (Object.keys(axes).length === 0) return; 142 | return axes; 143 | } 144 | 145 | getData(metricName, rawMetric, visualization) { 146 | const data = { populations: {} }; 147 | 148 | const xFieldName = this.getXFieldName(metricName); 149 | 150 | let yFields = []; 151 | let seriesFields = []; 152 | 153 | for (const columnName of Object.keys(visualization.options.columnMapping)) { 154 | const columnValue = visualization.options.columnMapping[columnName]; 155 | 156 | if (columnValue === 'y') { 157 | yFields.push(columnName); 158 | } else if (columnValue === 'series') { 159 | seriesFields.push(columnName); 160 | } 161 | } 162 | 163 | for (const row of rawMetric.query_result.data.rows) { 164 | const seriesFound = seriesFields.some(sn => { 165 | return Object.keys(row).includes(sn); 166 | }); 167 | 168 | if (seriesFound) { 169 | for (const seriesName of seriesFields) { 170 | const populationName = row[seriesName]; 171 | data.populations[populationName] = data.populations[populationName] || []; 172 | 173 | if (yFields.length !== 1) { 174 | throw new Error('If a series is found, "yFields" should have exactly one member'); 175 | } 176 | 177 | data.populations[populationName].push({ 178 | x: row[xFieldName], 179 | y: row[yFields[0]], 180 | }); 181 | } 182 | } else { 183 | const populations = yFields; 184 | for (const populationName of populations) { 185 | data.populations[populationName] = data.populations[populationName] || []; 186 | data.populations[populationName].push({ 187 | x: row[xFieldName], 188 | y: row[populationName], 189 | }); 190 | } 191 | } 192 | } 193 | 194 | return data; 195 | } 196 | 197 | getXFieldName(metricName, visualization) { 198 | if (!visualization) { 199 | visualization = this.getVisualization(metricName); 200 | } 201 | 202 | return Object.keys(visualization.options.columnMapping).find(columnName => { 203 | return visualization.options.columnMapping[columnName] === 'x'; 204 | }); 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | const startTime = process.hrtime(); 2 | 3 | const fs = require('fs'); 4 | const path = require('path'); 5 | const processData = require('./processData'); 6 | 7 | 8 | exports.default = async () => { 9 | const datasetConfigDirectory = path.join(__dirname, '../config/datasets'); 10 | const configFilenames = fs.readdirSync(datasetConfigDirectory); 11 | 12 | const processPromises = []; 13 | 14 | for (const configFilename of configFilenames) { 15 | const datasetName = configFilename.replace('.json', ''); 16 | const datasetConfig = JSON.parse( 17 | fs.readFileSync( 18 | path.join(datasetConfigDirectory, configFilename), 19 | 'utf8', 20 | ) 21 | ); 22 | 23 | processPromises.push( 24 | processData(datasetName, datasetConfig) 25 | ); 26 | } 27 | 28 | try { 29 | await Promise.all(processPromises); 30 | 31 | const timeDifference = process.hrtime(startTime); 32 | const nanosecondsInEachSecond = 1e9; 33 | const timePrecision = 5; 34 | 35 | // timeDifference is an array where the first element is the number of 36 | // seconds that have passed and the second element is the number of 37 | // *additional* nanoseconds that have passed 38 | const elapsedSeconds = timeDifference[0] + 39 | (timeDifference[1] / nanosecondsInEachSecond); 40 | 41 | // eslint-disable-next-line no-console 42 | console.log(`Wrote all files in ${elapsedSeconds.toPrecision(timePrecision)}s`); 43 | } catch (err) { 44 | if (err.stack) { 45 | // eslint-disable-next-line no-console 46 | console.error('Error:', err.stack); 47 | } else if (err.message) { 48 | // eslint-disable-next-line no-console 49 | console.error('Error:', err.message); 50 | } else { 51 | // eslint-disable-next-line no-console 52 | console.error('Error:', err); 53 | } 54 | 55 | // eslint-disable-next-line no-process-exit 56 | process.exit(1); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/processData.js: -------------------------------------------------------------------------------- 1 | require('dotenv').config(); 2 | 3 | const fetch = require('node-fetch'); 4 | const S3 = require('aws-sdk/clients/s3'); 5 | const {Storage} = require('@google-cloud/storage'); 6 | const fse = require('fs-extra'); 7 | 8 | const QuantumFormatter = require('./formatters/QuantumFormatter'); 9 | const BabbageFormatter = require('./formatters/BabbageFormatter'); 10 | const RedashFormatter = require('./formatters/RedashFormatter'); 11 | 12 | const timePrecision = 5; 13 | 14 | module.exports = async (datasetName, datasetConfig) => { 15 | for (const platform of Object.keys(datasetConfig.sources)) { 16 | const dataURL = parseDataURL(datasetConfig.sources[platform].data.url); 17 | const data = await getJSON(dataURL); 18 | 19 | let annotations; 20 | if (datasetConfig.sources[platform].annotations && datasetConfig.sources[platform].annotations.url) { 21 | annotations = await getJSON(datasetConfig.sources[platform].annotations.url); 22 | } 23 | 24 | const format = datasetConfig.sources[platform].data.format; 25 | 26 | let FormatterConstructor; 27 | switch(format) { 28 | case 'quantum': 29 | FormatterConstructor = QuantumFormatter; 30 | break; 31 | case 'babbage': 32 | FormatterConstructor = BabbageFormatter; 33 | break; 34 | case 'redash': 35 | FormatterConstructor = RedashFormatter; 36 | break; 37 | default: 38 | throw new Error(`Format "${format}" is not supported (set for platform ${platform} of dataset "${datasetName}")`); 39 | } 40 | 41 | const formatter = new FormatterConstructor(datasetName, platform, datasetConfig, data, annotations); 42 | 43 | const summary = await formatter.getSummary(); 44 | 45 | const writePromises = []; 46 | 47 | writePromises.push(new Promise((resolve, reject) => { 48 | writeData( 49 | `datasets/${platform}/${datasetName}/index.json`, 50 | JSON.stringify(summary), 51 | ).then(resolve).catch(reject); 52 | })); 53 | 54 | for (const categoryName of summary.categories) { 55 | for (const metricName of summary.metrics) { 56 | 57 | const filename = `datasets/${platform}/${datasetName}/${categoryName}/${metricName}/index.json`; 58 | const metric = await formatter.getMetric(categoryName, metricName); 59 | 60 | writePromises.push(new Promise((resolve, reject) => { 61 | writeData( 62 | filename, 63 | JSON.stringify(metric), 64 | ).then(resolve).catch(reject); 65 | })); 66 | 67 | } 68 | } 69 | 70 | await Promise.all(writePromises); 71 | 72 | formatter.clearCache(); 73 | } 74 | } 75 | 76 | function parseDataURL(urlConfig) { 77 | if (typeof urlConfig === 'string') return urlConfig; 78 | return urlConfig.base + '?' + Object.keys(urlConfig.query).map(q => { 79 | return `${q}=${process.env[urlConfig.query[q]]}`; 80 | }).join('&'); 81 | } 82 | 83 | async function getJSON(url) { 84 | const response = await fetch(url); 85 | return await response.json() 86 | } 87 | 88 | function writeData(filename, data) { 89 | return new Promise((resolve, reject) => { 90 | const startTime = process.hrtime(); 91 | var protocol; 92 | var bucketName; 93 | var uploadPromise; 94 | 95 | if (process.env.GCS_BUCKET_NAME !== undefined) { 96 | protocol = 'gs' 97 | bucketName = process.env.GCS_BUCKET_NAME; 98 | uploadPromise = new Storage().bucket(bucketName).file(filename).save(data, {contentType: 'application/json'}); 99 | } else if (process.env.AWS_BUCKET_NAME !== undefined) { 100 | protocol = 's3' 101 | bucketName = process.env.AWS_BUCKET_NAME; 102 | 103 | const objectParams = { 104 | Bucket: bucketName, 105 | Key: filename, 106 | Body: data, 107 | ContentType: 'application/json', 108 | }; 109 | 110 | uploadPromise = new S3({ 111 | apiVersion: '2006-03-01', 112 | region: process.env.AWS_REGION, 113 | accessKeyId: process.env.AWS_ACCESS_KEY_ID, 114 | secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, 115 | }).putObject(objectParams).promise(); 116 | } else { 117 | protocol = 'file' 118 | bucketName = `${process.cwd()}/target` 119 | uploadPromise = fse.outputFile(`${bucketName}/${filename}`, data); 120 | } 121 | 122 | uploadPromise.then(() => { 123 | const timeDifference = process.hrtime(startTime); 124 | const elapsedMilliseconds = timeDifferenceToMilliseconds(timeDifference); 125 | 126 | // eslint-disable-next-line no-console 127 | console.log(`[${new Date().toISOString()}] wrote ${protocol}://${bucketName}/${filename} in ${elapsedMilliseconds.toPrecision(timePrecision)}ms`); 128 | 129 | return resolve(); 130 | }).catch(err => { 131 | return reject(err); 132 | }); 133 | }); 134 | } 135 | 136 | function timeDifferenceToMilliseconds(timeDifference) { 137 | const millisecondsInEachSecond = 1000; 138 | const nanosecondsInEachMillisecond = 1e6; 139 | 140 | // timeDifference is an array where the first element is the number of 141 | // seconds that have passed and the second element is the number of 142 | // *additional* nanoseconds that have passed 143 | return ( 144 | timeDifference[0] * millisecondsInEachSecond 145 | ) + ( 146 | timeDifference[1] / nanosecondsInEachMillisecond 147 | ); 148 | } 149 | -------------------------------------------------------------------------------- /src/tests/special/api-equivalence.test.js: -------------------------------------------------------------------------------- 1 | const chai = require('chai'); 2 | const deepEqualInAnyOrder = require('deep-equal-in-any-order'); 3 | 4 | const utils = require('../utils'); 5 | 6 | chai.use(deepEqualInAnyOrder); 7 | 8 | 9 | it('Development API output is equivalent to production API output', function(done) { 10 | this.timeout(50000); 11 | 12 | async function compare() { 13 | const datasetNames = utils.getDatasetNames(); 14 | 15 | for (const datasetName of datasetNames) { 16 | 17 | for (const platform of await utils.getPlatforms(datasetName)) { 18 | const developmentSummary = await utils.getDevelopmentJSON(`${platform}/${datasetName}`); 19 | const productionSummary = await utils.getProductionJSON(`${platform}/${datasetName}`); 20 | 21 | chai.expect(productionSummary).to.deep.equalInAnyOrder(developmentSummary); 22 | 23 | // The order of the dates *does* matter 24 | chai.expect(productionSummary.dates).to.deep.equal(developmentSummary.dates); 25 | 26 | for (const categoryName of developmentSummary.categories) { 27 | for (const metricName of developmentSummary.metrics) { 28 | const metricPath = `${platform}/${datasetName}/${categoryName}/${metricName}`; 29 | const developmentMetric = await utils.getDevelopmentJSON(metricPath, platform); 30 | const productionMetric = await utils.getProductionJSON(metricPath, platform); 31 | chai.expect(productionMetric).to.deep.equalInAnyOrder(developmentMetric); 32 | } 33 | } 34 | } 35 | } 36 | } 37 | 38 | compare().then(done).catch(done); 39 | }); 40 | -------------------------------------------------------------------------------- /src/tests/standard/summary.test.js: -------------------------------------------------------------------------------- 1 | const { assert } = require('chai'); 2 | const utils = require('../utils'); 3 | 4 | 5 | it('All dates are in descending order', function(done) { 6 | async function testDatasets() { 7 | const datasetNames = utils.getDatasetNames(); 8 | 9 | for (const datasetName of datasetNames) { 10 | for (const platform of await utils.getPlatforms(datasetName)) { 11 | const localSummary = await utils.getDevelopmentJSON(`${platform}/${datasetName}`); 12 | const returnedDates = localSummary.dates; 13 | const correctlySortedDates = returnedDates.concat().sort().reverse(); 14 | 15 | assert.deepEqual(returnedDates, correctlySortedDates); 16 | } 17 | } 18 | } 19 | 20 | testDatasets().then(done).catch(done); 21 | }); 22 | -------------------------------------------------------------------------------- /src/tests/utils.js: -------------------------------------------------------------------------------- 1 | const fetch = require("node-fetch"); 2 | const fs = require('fs'); 3 | const path = require('path'); 4 | const { promisify } = require('util'); 5 | 6 | 7 | const readFilePromisified = promisify(fs.readFile); 8 | 9 | function getDatasetNames() { 10 | const datasetConfigDirectory = path.join( 11 | __dirname, 12 | '../../config/datasets' 13 | ); 14 | const filenames = fs.readdirSync(datasetConfigDirectory); 15 | const datasetNames = []; 16 | 17 | for (const filename of filenames) { 18 | datasetNames.push(filename.replace('.json', '')); 19 | } 20 | 21 | return datasetNames; 22 | } 23 | 24 | async function getPlatforms(datasetName) { 25 | const datasetConfig = JSON.parse(await readFilePromisified(path.join( 26 | __dirname, 27 | `../../config/datasets/${datasetName}.json` 28 | ), 'utf-8')); 29 | 30 | return Object.keys(datasetConfig.sources); 31 | } 32 | 33 | async function getDevelopmentJSON(identifier) { 34 | const Body = await readFilePromisified(`target/datasets/${identifier}/index.json`); 35 | return JSON.parse(Body); 36 | } 37 | 38 | async function getProductionJSON(identifier) { 39 | const response = await fetch(`https://data.firefox.com/datasets/${identifier}/index.json`); 40 | return await response.json(); 41 | } 42 | 43 | module.exports = { 44 | getDatasetNames, 45 | getPlatforms, 46 | getDevelopmentJSON, 47 | getProductionJSON, 48 | }; 49 | --------------------------------------------------------------------------------