├── .circleci
    └── config.yml
├── .dockerignore
├── .gitignore
├── .pyup.yml
├── CODE_OF_CONDUCT.md
├── Dockerfile
├── LICENSE
├── Makefile
├── README.md
├── bin
    ├── deploy-dockerhub.sh
    ├── deployment-bug.py
    ├── make-release.py
    ├── run.sh
    ├── sanspreamble.py
    ├── upload_to_s3.py
    └── wait-for
├── docker-compose.ci.yml
├── docker-compose.yml
├── docs
    ├── Dockerfile
    ├── Makefile
    ├── _static
    │   └── theme_overrides.css
    ├── api.rst
    ├── conf.py
    ├── configuration.rst
    ├── dev.rst
    ├── index.rst
    ├── jobs.rst
    ├── lambda-1.png
    ├── lambda-2.png
    ├── lambda-3.png
    ├── lambda-4.png
    ├── overview.png
    ├── requirements.txt
    ├── run.sh
    └── support.rst
├── jobs
    ├── CHANGELOG.rst
    ├── CONTRIBUTORS.rst
    ├── MANIFEST.in
    ├── README.rst
    ├── buildhub
    │   ├── __init__.py
    │   ├── configure_markus.py
    │   ├── initialization.yml
    │   ├── inventory_to_records.py
    │   ├── lambda_s3_event.py
    │   ├── s3_inventory_to_kinto.py
    │   ├── to_kinto.py
    │   └── utils.py
    ├── requirements
    │   ├── constraints.txt
    │   ├── default.txt
    │   └── dev.txt
    ├── setup.cfg
    ├── setup.py
    └── tests
    │   ├── data
    │       ├── inventory-simple.csv
    │       └── s3-event-simple.json
    │   ├── test_inventory_to_records.py
    │   ├── test_inventory_to_records_functional.py
    │   ├── test_lambda_s3_event.py
    │   ├── test_lambda_s3_event_functional.py
    │   ├── test_s3_inventory_to_kinto.py
    │   ├── test_to_kinto.py
    │   └── test_utils.py
├── kinto
    ├── Dockerfile
    ├── README.md
    ├── kinto.ini
    ├── requirements.txt
    └── run.sh
├── testkinto
    ├── Dockerfile
    ├── README.md
    ├── kinto.ini
    ├── requirements.txt
    └── run.sh
└── ui
    ├── .gitignore
    ├── Dockerfile
    ├── README.md
    ├── lint_problems.sh
    ├── package.json
    ├── public
        ├── favicon.ico
        ├── index.html
        └── manifest.json
    ├── run.sh
    ├── src
        ├── App.css
        ├── App.js
        ├── App.test.js
        ├── contribute.json
        ├── index.css
        ├── index.js
        └── registerServiceWorker.js
    └── yarn.lock


/.circleci/config.yml:
--------------------------------------------------------------------------------
  1 | # These environment variables must be set in CircleCI UI
  2 | #
  3 | # DOCKERHUB_REPO - docker hub repo, format: <username>/<repo>
  4 | # DOCKER_EMAIL   - login info for docker hub
  5 | # DOCKER_USER
  6 | # DOCKER_PASS
  7 | #
  8 | 
  9 | version: 2
 10 | jobs:
 11 |   checkout_code:
 12 |     docker:
 13 |       - image: ubuntu:16.04
 14 |     working_directory: ~/buildhub
 15 |     steps:
 16 |       - run:
 17 |           name: Install essential packages
 18 |           command: |
 19 |             apt-get update && apt-get install -y ca-certificates git
 20 |       - checkout
 21 |       - save_cache:
 22 |           key: v1-repo-{{ .Environment.CIRCLE_SHA1 }}
 23 |           paths:
 24 |             - ~/buildhub
 25 | 
 26 |   build_test_and_deploy:
 27 |     docker:
 28 |       - image: ubuntu:16.04
 29 |     working_directory: ~/buildhub
 30 |     steps:
 31 |       - run:
 32 |           name: Install essential packages
 33 |           command: |
 34 |             apt-get update && apt-get install -y ca-certificates curl python3-pip zip
 35 | 
 36 |       - restore_cache:
 37 |           keys:
 38 |             - v1-repo-{{ .Environment.CIRCLE_SHA1 }}
 39 | 
 40 |       - run:
 41 |           name: Install Docker
 42 |           command: |
 43 |             set -x
 44 |             VER="17.09.0-ce"
 45 |             curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz
 46 |             tar -xz -C /tmp -f /tmp/docker-$VER.tgz
 47 |             mv /tmp/docker/* /usr/bin
 48 | 
 49 |       - run:
 50 |           name: Install Docker Compose
 51 |           command: |
 52 |             set -x
 53 |             curl -L https://github.com/docker/compose/releases/download/1.16.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose
 54 |             chmod +x /usr/local/bin/docker-compose
 55 | 
 56 |       - setup_remote_docker:
 57 |           version: 17.09.0-ce
 58 | 
 59 |       - run:
 60 |           name: Create version.json
 61 |           command: |
 62 |             # create a version.json per https://github.com/mozilla-services/Dockerflow/blob/master/docs/version_object.md
 63 |             printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \
 64 |             "$CIRCLE_SHA1" \
 65 |             "$CIRCLE_TAG" \
 66 |             "$CIRCLE_PROJECT_USERNAME" \
 67 |             "$CIRCLE_PROJECT_REPONAME" \
 68 |             "$CIRCLE_BUILD_URL" > version.json
 69 | 
 70 |       - run:
 71 |           name: Start testkinto (an in-memory only kinto server)
 72 |           command: |
 73 |             # This assures that the `docker-compose up`, later, isn't
 74 |             # put to do the build in the background if it needs to be built.
 75 |             # Basically, by being explicit about it here and now we can
 76 |             # we certain the image is ready by the time we start it ("up")
 77 |             # in the background ("-d").
 78 |             docker-compose -f docker-compose.ci.yml build testkinto
 79 | 
 80 |             # Use -d to put it in the background "docker style".
 81 |             docker-compose -f docker-compose.ci.yml up -d testkinto
 82 | 
 83 |             # Give the 'up -d testkinto' a fair chance to get started
 84 |             sleep 10
 85 | 
 86 |       - run:
 87 |           name: Run functional tests
 88 |           command: |
 89 |             docker-compose -f docker-compose.ci.yml run buildhub waitfor testkinto:9999 -t 30
 90 |             docker-compose -f docker-compose.ci.yml run testkinto initialize-kinto-wizard jobs/buildhub/initialization.yml  --server http://testkinto:9999/v1 --auth user:pass
 91 |             docker-compose -f docker-compose.ci.yml run buildhub functional-tests
 92 | 
 93 |       - run:
 94 |           name: Lint check
 95 |           command: |
 96 |             docker-compose -f docker-compose.ci.yml run buildhub lintcheck
 97 |             docker-compose -f docker-compose.ci.yml run ui lintcheck
 98 | 
 99 |       - run:
100 |           name: Mock run building lambda.zip.
101 |           command: |
102 |             docker-compose -f docker-compose.ci.yml run buildhub lambda.zip
103 | 
104 |       - run:
105 |           name: Docs should build without error
106 |           command: |
107 |             docker-compose -f docker-compose.ci.yml run docs build
108 | 
109 |       - run:
110 |           name: Push to Dockerhub
111 |           command: |
112 |             # set DOCKER_DEPLOY=true in Circle UI to push to Dockerhub
113 |             DOCKER_DEPLOY="${DOCKER_DEPLOY:-false}"
114 |             if [ "${CIRCLE_BRANCH}" == "master" ]; then
115 |               bin/deploy-dockerhub.sh latest
116 |             fi
117 |             if [ -n "${CIRCLE_TAG}" ]; then
118 |               bin/deploy-dockerhub.sh "$CIRCLE_TAG"
119 |             fi
120 | 
121 |   frontend:
122 |     docker:
123 |       - image: node:9
124 |     working_directory: ~/buildhub
125 |     steps:
126 |       - checkout
127 | 
128 |       - run:
129 |           name: "Checking Versions"
130 |           command: |
131 |             node --version
132 |             yarn --version
133 | 
134 |       - run:
135 |           name: Install frontend dependencies
136 |           command: |
137 |             cd ui
138 |             yarn
139 | 
140 |       - run:
141 |           name: Build frontend
142 |           command: |
143 |             cd ui
144 |             yarn run build
145 | 
146 |       - store_artifacts:
147 |           path: ui/build
148 | 
149 | workflows:
150 |   version: 2
151 | 
152 |   # workflow jobs are _not_ run in tag builds by default
153 |   # we use filters to whitelist jobs that should be run for tags
154 | 
155 |   # workflow jobs are run in _all_ branch builds by default
156 |   # we use filters to blacklist jobs that shouldn't be run for a branch
157 | 
158 |   # see: https://circleci.com/docs/2.0/workflows/#git-tag-job-execution
159 | 
160 |   build-test-deploy:
161 |     jobs:
162 |       - checkout_code:
163 |           filters:
164 |             tags:
165 |               only: /.*/
166 |       - build_test_and_deploy:
167 |           requires:
168 |             - checkout_code
169 |           filters:
170 |             tags:
171 |               only: /.*/
172 |       - frontend
173 | 


--------------------------------------------------------------------------------
/.dockerignore:
--------------------------------------------------------------------------------
 1 | .git
 2 | buildhub-lambda*.zip
 3 | **/__pycache__
 4 | *.pyc
 5 | *.egg-info
 6 | .venv
 7 | *.zip
 8 | 
 9 | jobs/.cache
10 | ui/node_modules/
11 | csv-download-directory/
12 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | .cache/
 2 | .coverage/
 3 | .tox/
 4 | *.egg-info/
 5 | __pycache__/
 6 | jobs/.coverage
 7 | docs/_build/
 8 | .venv/
 9 | lambda.zip
10 | settings.ini
11 | .env
12 | version.json
13 | .bash_history
14 | buildhub-lambda-*.zip
15 | .docker-build
16 | .metadata-*.json
17 | csv-download-directory/
18 | .records-hashes-kinto-build-hub-releases.json
19 | .pytest_cache/
20 | .manifest-*.json
21 | 


--------------------------------------------------------------------------------
/.pyup.yml:
--------------------------------------------------------------------------------
 1 | search: False
 2 | label_prs: update
 3 | requirements:
 4 |   - jobs/requirements/default.txt:
 5 |       update: insecure
 6 |   - jobs/requirements/constraints.txt:
 7 |       update: insecure
 8 |   - jobs/requirements/dev.txt:
 9 |       update: insecure
10 |   - docs/requirements.txt:
11 |       pin: False
12 |       update: insecure
13 | 


--------------------------------------------------------------------------------
/CODE_OF_CONDUCT.md:
--------------------------------------------------------------------------------
 1 | # Community Participation Guidelines
 2 | 
 3 | This repository is governed by Mozilla's code of conduct and etiquette guidelines. 
 4 | For more details, please read the
 5 | [Mozilla Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/). 
 6 | 
 7 | ## How to Report
 8 | For more information on how to report violations of the Community Participation Guidelines, please read our '[How to Report](https://www.mozilla.org/about/governance/policies/participation/reporting/)' page.
 9 | 
10 | <!--
11 | ## Project Specific Etiquette
12 | 
13 | In some cases, there will be additional project etiquette i.e.: (https://bugzilla.mozilla.org/page.cgi?id=etiquette.html).
14 | Please update for your project.
15 | -->
16 | 


--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM python:3.6-slim
 2 | MAINTAINER Product Delivery irc://irc.mozilla.org/#storage-team
 3 | 
 4 | ENV PYTHONUNBUFFERED=1 \
 5 |     PYTHONPATH=/app/
 6 | 
 7 | # add a non-privileged user for installing and running the application
 8 | # don't use --create-home option to prevent populating with skeleton files
 9 | RUN mkdir /app && \
10 |     chown 10001:10001 /app && \
11 |     groupadd --gid 10001 app && \
12 |     useradd --no-create-home --uid 10001 --gid 10001 --home-dir /app app
13 | 
14 | # install a few essentials and clean apt caches afterwards
15 | RUN apt-get update && \
16 |     apt-get install -y --no-install-recommends \
17 |         apt-transport-https build-essential curl git zip netcat
18 | 
19 | # Clean up apt
20 | RUN apt-get autoremove -y && \
21 |     apt-get clean && \
22 |     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
23 | 
24 | COPY . /app
25 | 
26 | # Install Python dependencies
27 | RUN pip install -e app/jobs/
28 | COPY jobs/requirements /tmp/requirements
29 | # Switch to /tmp to install dependencies outside home dir
30 | WORKDIR /tmp
31 | RUN ls /tmp
32 | RUN pip install --no-cache-dir \
33 |   -r requirements/default.txt \
34 |   -r requirements/dev.txt \
35 |   -c requirements/constraints.txt
36 | 
37 | COPY . /app
38 | 
39 | # Switch back to home directory
40 | WORKDIR /app
41 | 
42 | 
43 | RUN chown -R 10001:10001 /app
44 | 
45 | USER 10001
46 | 
47 | 
48 | # Using /bin/bash as the entrypoint works around some volume mount issues on Windows
49 | # where volume-mounted files do not have execute bits set.
50 | # https://github.com/docker/compose/issues/2301#issuecomment-154450785 has additional background.
51 | ENTRYPOINT ["/bin/bash", "/app/bin/run.sh"]
52 | 
53 | CMD ["help"]
54 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | # This Source Code Form is subject to the terms of the Mozilla Public
 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this
 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 4 | 
 5 | .PHONY: help
 6 | help: default
 7 | 
 8 | .PHONY: default
 9 | default:
10 | 	@echo "Welcome to Buildhub\n"
11 | 	@echo "  build                       build buildhub Docker image"
12 | 	@echo "  run                         start everything"
13 | 	@echo "  clean                       delete local files"
14 | 	@echo "  stop                        stop any docker containers"
15 | 	@echo ""
16 | 	@echo "  functional-tests            run the functional tests"
17 | 	@echo "  unit-tests                  run the pure python unit tests"
18 | 	@echo "  lambda.zip                  build lambda.zip from within the container"
19 | 	@echo "  lintcheck                   run lint checking (i.e. flake8)"
20 | 	@echo "  upload-to-s3                upload lambda.zip to AWS"
21 | 	@echo "  shell                       enter a bash shell with volume mount"
22 | 	@echo "  sudo-shell                  enter a bash shell, as root, with volume mount"
23 | 	@echo "  psql                        open the psql cli"
24 | 	@echo "\n"
25 | 
26 | .docker-build:
27 | 	make build
28 | 
29 | .PHONY: clean
30 | clean:
31 | 	rm -fr lambda.zip
32 | 	rm -fr .docker-build
33 | 	rm -fr .metadata*.json
34 | 
35 | .PHONY: run
36 | run:
37 | 	docker-compose up
38 | 
39 | .PHONY: stop
40 | stop:
41 | 	docker-compose stop
42 | 
43 | .PHONY: build
44 | build:
45 | 	docker-compose build buildhub
46 | 	touch .docker-build
47 | 
48 | .PHONY: functional-tests
49 | functional-tests:
50 | 	docker-compose up -d testkinto
51 | 	./bin/wait-for localhost:9999
52 | 	docker-compose run kinto initialize-kinto-wizard jobs/buildhub/initialization.yml  --server http://testkinto:9999/v1 --auth user:pass
53 | 	docker-compose run buildhub functional-tests
54 | 	docker-compose stop
55 | 
56 | .PHONY: unit-tests
57 | unit-tests: .docker-build
58 | 	docker-compose run buildhub unit-tests
59 | 
60 | lambda.zip: .docker-build
61 | 	docker-compose run buildhub lambda.zip
62 | 
63 | .PHONY: upload-to-s3
64 | upload-to-s3:
65 | 	echo "NotImplemented" && exit 1
66 | 	# $(PYTHON) bin/upload_to_s3.py
67 | 
68 | .PHONY: docs
69 | docs:
70 | 	docker-compose run docs build
71 | 
72 | .PHONY: lintcheck
73 | lintcheck: .docker-build
74 | 	docker-compose run buildhub lintcheck
75 | 	# docker-compose run ui lintcheck
76 | 
77 | .PHONY: shell
78 | shell:
79 | 	docker-compose run buildhub bash
80 | 
81 | .PHONY: sudo-shell
82 | sudo-shell:
83 | 	docker-compose run --user 0 buildhub bash
84 | 
85 | .PHONY: psql
86 | psql:
87 | 	docker-compose run db psql -h db -U postgres
88 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Buildhub
  2 | 
  3 | ## Status: June 11th, 2019
  4 | 
  5 | This project is deprecated and will be decommissioned soon.
  6 | If you're using Buildhub, please migrate to [Buildhub2](https://buildhub2.readthedocs.io/).
  7 | 
  8 | ## Details
  9 | 
 10 | [![CircleCI](https://circleci.com/gh/mozilla-services/buildhub.svg?style=svg)](https://circleci.com/gh/mozilla-services/buildhub)
 11 | 
 12 | _Buildhub_ aims to provide a public database of comprehensive information about releases and builds.
 13 | 
 14 | * [Online catalog](https://mozilla-services.github.io/buildhub/)
 15 | * [Web API documentation](https://buildhub.readthedocs.io)
 16 | 
 17 | ## Licence
 18 | 
 19 | [MPL 2.0](http://www.mozilla.org/MPL/2.0/)
 20 | 
 21 | ## Development
 22 | 
 23 | 1.  Install Docker
 24 | 2.  To run tests: `make test`
 25 | 3.  To lint check Python code: `make lintcheck`
 26 | 
 27 | ## Continuous Integration
 28 | 
 29 | We use [CircleCI](https://circleci.com/gh/mozilla-services/buildhub)
 30 | for all continous integration.
 31 | 
 32 | ## Releasing
 33 | 
 34 | There are a few pieces to Buildhub.
 35 | 
 36 | ### AWS Lambda job and cron job
 37 | 
 38 | Generate a new `lambda.zip` file by running:
 39 | 
 40 |     rm lambda.zip
 41 |     make lambda.zip
 42 | 
 43 | This runs a script inside a Docker container to generate the `lambda.zip`
 44 | file.
 45 | 
 46 | You need to have write access to `github.com/mozilla-services/buildhub`.
 47 | 
 48 | You need a [GitHub Personal Access Token](https://github.com/settings/tokens)
 49 | with `repos` scope. This is to generate GitHub Releases and upload assets
 50 | to them.
 51 | 
 52 | Create a Python virtual environment and install "requests" and "python-decouple"
 53 | into it.
 54 | 
 55 | Run `./bin/make-release.py`. You need to set the `GITHUB_API_KEY` environment
 56 | variable. You need to specify the "type" of the release as a command-line
 57 | argument. Choices are:
 58 | 
 59 | * `major` (e.g. '2.6.9' to '3.0.0')
 60 | * `minor` (e.g. '2.6.7' to '2.7.0')
 61 | * `patch` (e.g. '2.6.7' to '2.6.8')
 62 | 
 63 | Then do this in your Python virtual environment:
 64 | 
 65 |     $ GITHUB_API_KEY=895f...ce09 ./bin/make-release.py minor
 66 | 
 67 | This will bump the version in `setup.py`, update the `CHANGELOG.rst` and
 68 | make a tag and push that tag to GitHub.
 69 | 
 70 | Then, it will create a Release and upload the latest `lambda.zip` as an
 71 | attachment to that Release.
 72 | 
 73 | You need to file a Bugzilla bug to have the Lambda job upgraded on Stage.
 74 | [Issue #423](https://github.com/mozilla-services/buildhub/issues/423)
 75 | is about automating this away.
 76 | 
 77 | To upgrade the Lambda job on **Stage** run:
 78 | 
 79 |     ./bin/deployment-bug.py stage-lambda
 80 | 
 81 | To upgrade the cron job _and_ Lambda job on **Prod** run:
 82 | 
 83 |     ./bin/deployment-bug.py prod
 84 | 
 85 | ### Website ui
 86 | 
 87 | Install yarn.
 88 | 
 89 | Then run:
 90 | 
 91 |     $ cd ui
 92 |     $ yarn install
 93 |     $ yarn run build
 94 |     $ rimraf tmp
 95 |     $ mkdir tmp
 96 |     $ cp -R build/* tmp/
 97 |     $ gh-pages -d tmp --add
 98 | 
 99 | Note: This only deploys a ui that connects to prod kinto--it doesn't deploy a
100 | ui that connects to the stage kinto.
101 | 
102 | ## Datadog
103 | 
104 | [Buildhub Performance](https://app.datadoghq.com/dash/794559/buildhub-performance)
105 | 


--------------------------------------------------------------------------------
/bin/deploy-dockerhub.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # THIS IS MEANT TO BE RUN BY CI
 4 | 
 5 | set -e
 6 | 
 7 | # Usage: retry MAX CMD...
 8 | # Retry CMD up to MAX times. If it fails MAX times, returns failure.
 9 | # Example: retry 3 docker push "mozilla/buildhub:$TAG"
10 | function retry() {
11 |     max=$1
12 |     shift
13 |     count=1
14 |     until "$@"; do
15 |         count=$((count + 1))
16 |         if [[ $count -gt $max ]]; then
17 |             return 1
18 |         fi
19 |         echo "$count / $max"
20 |     done
21 |     return 0
22 | }
23 | 
24 | # configure docker creds
25 | retry 3  echo "$DOCKER_PASS" | docker login -u="$DOCKER_USER" --password-stdin
26 | 
27 | # docker tag and push git branch to dockerhub
28 | if [ -n "$1" ]; then
29 |     [ "$1" == master ] && TAG=latest || TAG="$1"
30 |     docker tag "$DOCKERHUB_REPO" "$DOCKERHUB_REPO:$TAG" ||
31 |         (echo "Couldn't tag $DOCKERHUB_REPO as $DOCKERHUB_REPO:$TAG" && false)
32 |     retry 3 docker push "$DOCKERHUB_REPO:$TAG" ||
33 |         (echo "Couldn't $DOCKERHUB_REPO:$TAG" && false)
34 |     echo "Pushed $DOCKERHUB_REPO:$TAG"
35 | fi
36 | 


--------------------------------------------------------------------------------
/bin/deployment-bug.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # This Source Code Form is subject to the terms of the Mozilla Public
  3 | # License, v. 2.0. If a copy of the MPL was not distributed with this
  4 | # file, you can obtain one at http://mozilla.org/MPL/2.0/.
  5 | 
  6 | from urllib.parse import urlencode
  7 | 
  8 | import requests
  9 | 
 10 | 
 11 | OWNER = 'mozilla-services'
 12 | REPO = 'buildhub'
 13 | 
 14 | VALID_ENVIRONMENTS = ('stage', 'prod')
 15 | VALID_TASKS = ('cron', 'lambda', 'both')
 16 | 
 17 | QA_CONTACT = 'chartjes@mozilla.com'
 18 | 
 19 | 
 20 | def main(
 21 |     environment,
 22 |     task,
 23 |     tag=None,
 24 |     dry_run=False,
 25 | ):
 26 |     if environment == 'stage':
 27 |         print("""
 28 | ⚠️  NOTE! ⚠️
 29 | 
 30 | Stage is automatically upgraded (both cron and Lambda) when a new GitHub
 31 | release is made. Are you sure you need this bug?
 32 |         """)
 33 |         if not input("Sure? [y/N] ").strip().lower() in ('yes', 'y'):
 34 |             print("Thought so.")
 35 |             return 5
 36 | 
 37 |     api_url = f'https://api.github.com/repos/{OWNER}/{REPO}/releases'
 38 |     if not tag:
 39 |         api_url += '/latest'
 40 |     else:
 41 |         api_url += f'/tags/{tag}'
 42 |     response = requests.get(api_url)
 43 |     response.raise_for_status()
 44 |     release_info = response.json()
 45 | 
 46 |     # Prepare some variables for the string templates
 47 |     release_url = release_info['html_url']
 48 |     lambda_asset_url = None
 49 |     for asset in release_info['assets']:
 50 |         lambda_asset_url = asset['browser_download_url']
 51 |     env = environment.title()
 52 |     release_tag_name = release_info['tag_name']
 53 | 
 54 |     if task == 'lambda':
 55 |         summary = f"On {env}, please deploy Buildhub Lambda function {release_tag_name}"  # noqa
 56 |         comment = f"""
 57 |             Could you please update the Lambda function for Buildhub {env} with the following one?
 58 | 
 59 |             {release_url}
 60 | 
 61 |             {lambda_asset_url}
 62 | 
 63 |             Thanks!
 64 |         """  # noqa
 65 |     elif task == 'cron':
 66 |         summary = f"On {env}, please deploy Buildhub Cron function {release_tag_name}"  # noqa
 67 |         comment = f"""
 68 |             Could you please update the Cron function for Buildhub {env} with the following one?
 69 | 
 70 |             {release_url}
 71 | 
 72 |             Thanks!
 73 |         """  # noqa
 74 |     else:
 75 |         summary = f"On {env}, please deploy Buildhub Cron and Lambda function {release_tag_name}"  # noqa
 76 |         comment = f"""
 77 |             Could you please update the Cron *and* Lambda function for Buildhub {env} with the following one?
 78 | 
 79 |             {release_url}
 80 | 
 81 |             {lambda_asset_url}
 82 | 
 83 |             Thanks!
 84 |         """  # noqa
 85 | 
 86 |     comment = '\n'.join(x.strip() for x in comment.strip().splitlines())
 87 |     params = {
 88 |         'qa_contact': QA_CONTACT,
 89 |         'comment': comment,
 90 |         'short_desc': summary,
 91 |         'component': 'Operations: Storage',
 92 |         'product': 'Cloud Services',
 93 |         'bug_file_loc': release_url,
 94 |     }
 95 |     URL = 'https://bugzilla.mozilla.org/enter_bug.cgi?' + urlencode(params)
 96 |     print('To file this bug, click (or copy) this URL:')
 97 |     print('👇')
 98 |     print(URL)
 99 |     print('👆')
100 |     return 0
101 | 
102 | 
103 | if __name__ == '__main__':
104 |     import argparse
105 | 
106 |     def check_environment(value):
107 |         value = value.strip()
108 |         if value not in VALID_ENVIRONMENTS:
109 |             raise argparse.ArgumentTypeError(
110 |                 f'{value!r} not in {VALID_ENVIRONMENTS}'
111 |             )
112 |         return value
113 | 
114 |     def check_task(value):
115 |         value = value.strip()
116 |         if value not in VALID_TASKS:
117 |             raise argparse.ArgumentTypeError(
118 |                 f'{value!r} not in {VALID_TASKS}'
119 |             )
120 |         return value
121 | 
122 |     parser = argparse.ArgumentParser(
123 |         description='Deploy Buildhub (by filing Bugzilla bugs)!'
124 |     )
125 |     parser.add_argument(
126 |         '-t', '--tag', type=str,
127 |         help=(
128 |             f'Name of the release (e.g. "v1.2.0"). If ommitted will be looked '
129 |             f'on GitHub at https://github.com/{OWNER}/{REPO}/releases'
130 |         )
131 |     )
132 | 
133 |     parser.add_argument(
134 |         '-e', '--environment',
135 |         type=check_environment,
136 |         default='prod',
137 |         help="Environment e.g. 'stage' or 'prod'. (Default 'prod')"
138 |     )
139 |     parser.add_argument(
140 |         'task', help='cron or lambda or both', type=check_task,
141 |     )
142 |     parser.add_argument('-d', '--dry-run', action='store_true')
143 |     args = parser.parse_args()
144 |     args = vars(args)
145 |     main(args.pop('environment'), args.pop('task'), **args)
146 | 


--------------------------------------------------------------------------------
/bin/make-release.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # This Source Code Form is subject to the terms of the Mozilla Public
  3 | # License, v. 2.0. If a copy of the MPL was not distributed with this
  4 | # file, you can obtain one at http://mozilla.org/MPL/2.0/.
  5 | 
  6 | import os
  7 | import datetime
  8 | import re
  9 | import subprocess
 10 | import time
 11 | 
 12 | import requests
 13 | from decouple import config
 14 | 
 15 | OWNER = 'mozilla-services'
 16 | REPO = 'buildhub'
 17 | 
 18 | GITHUB_API_KEY = config('GITHUB_API_KEY', default='')
 19 | 
 20 | 
 21 | def get_current_version():
 22 |     with open("jobs/setup.py", "r") as fp:
 23 |         lines = fp.readlines()
 24 | 
 25 |     for line in lines:
 26 |         if line.strip().startswith("version="):
 27 |             version = line.strip().replace("version=", "").replace("\'", "").replace(",", "")
 28 |             return version
 29 | 
 30 |     raise Exception("Can't find version.")
 31 | 
 32 | 
 33 | def _format_age(seconds):
 34 |     seconds = int(seconds)
 35 |     if seconds > 3600:
 36 |         return '{} hours {} minutes ago'.format(
 37 |             seconds // 3600,
 38 |             round(seconds % 3600 / 60),
 39 |         )
 40 |     elif seconds > 60:
 41 |         return '{} minutes {} seconds ago'.format(
 42 |             seconds // 60,
 43 |             round(seconds % 60),
 44 |         )
 45 |     else:
 46 |         return '{} seconds ago'.format(seconds)
 47 | 
 48 | 
 49 | def _format_file_size(bytes):
 50 |     if bytes > 1024 * 1024:
 51 |         return '{:.1f}MB'.format(bytes / 1024 / 1024)
 52 |     elif bytes > 1024:
 53 |         return '{:.1f}KB'.format(bytes / 1024)
 54 |     else:
 55 |         return '{}B'.format(bytes)
 56 | 
 57 | 
 58 | def check_output(*args, **kwargs):
 59 |     if len(args) == 1:
 60 |         if isinstance(args[0], str):
 61 |             args = args[0].split()
 62 |         else:
 63 |             args = args[0]
 64 |     return subprocess.check_output(args, **kwargs).decode('utf-8').strip()
 65 | 
 66 | 
 67 | def main(
 68 |     part,
 69 |     dry_run=False,
 70 |     github_api_key=None,
 71 |     tag_name_format='v{version}',
 72 |     upstream_name='master',
 73 | ):
 74 |     github_api_key = github_api_key or GITHUB_API_KEY
 75 |     assert github_api_key, 'GITHUB_API_KEY or --github-api-key not set.'
 76 | 
 77 |     # If this 401 errors, go here to generate a new personal access token:
 78 |     # https://github.com/settings/tokens
 79 |     # Give it all the 'repos' scope.
 80 |     api_url = 'https://api.github.com/user'
 81 |     response = requests.get(api_url, headers={
 82 |         'Authorization': 'token {}'.format(github_api_key)
 83 |     })
 84 |     response.raise_for_status()
 85 | 
 86 |     # Before we proceed, check that the `lambda.zip` is up to date.
 87 |     lambda_mtime = os.stat('lambda.zip').st_mtime
 88 |     age = time.time() - lambda_mtime
 89 |     print("The lambda.zip...\n")
 90 |     dt = datetime.datetime.fromtimestamp(lambda_mtime)
 91 |     print('\tLast modified:', dt.strftime('%d %B %Y - %H:%M:%S'))
 92 |     print('\tAge:', _format_age(age))
 93 |     print('')
 94 |     try:
 95 |         ok = input('Is this lambda.zip recently generated? [Y/n] ')
 96 |     except KeyboardInterrupt:
 97 |         print('\n\nTip! Generate it by running: make lambda.zip ')
 98 |         return 3
 99 |     if ok.lower().strip() == 'n':
100 |         print('Tip! Generate it by running: make lambda.zip ')
101 |         return 3
102 | 
103 |     # Figure out the current version
104 |     current_version = get_current_version()
105 |     z, y, x = [int(x) for x in current_version.split('.')]
106 |     if part == 'major':
107 |         next_version = (z + 1, 0, 0)
108 |     elif part == 'minor':
109 |         next_version = (z, y + 1, 0)
110 |     else:
111 |         next_version = (z, y, x + 1)
112 |     next_version = '.'.join(str(n) for n in next_version)
113 | 
114 |     # Figure out the CHANGELOG
115 | 
116 |     # Let's make sure we're up-to-date
117 |     current_branch = check_output('git rev-parse --abbrev-ref HEAD')
118 |     if current_branch != 'master':
119 |         # print("WARNING, NOT ON MASTER BRANCH")# DELETE WHEN DONE HACKING
120 |         print("Must be on the master branch to do this")
121 |         return 1
122 | 
123 |     # The current branch can't be dirty
124 |     try:
125 |         subprocess.check_call(
126 |             'git diff --quiet --ignore-submodules HEAD'.split()
127 |         )
128 |     except subprocess.CalledProcessError:
129 |         print(
130 |             "Can't be \"git dirty\" when we're about to git pull. "
131 |             "Stash or commit what you're working on."
132 |         )
133 |         return 2
134 | 
135 |     # Make sure we have all the old git tags
136 |     check_output(
137 |         f'git pull {upstream_name} master --tags',
138 |         stderr=subprocess.STDOUT
139 |     )
140 | 
141 |     # We're going to use the last tag to help you write a tag message
142 |     last_tag, last_tag_message = check_output([
143 |         'git',
144 |         'for-each-ref',
145 |         '--sort=-taggerdate',
146 |         '--count=1',
147 |         '--format',
148 |         '%(tag)|%(contents:subject)',
149 |         'refs/tags'
150 |     ]).split('|', 1)
151 | 
152 |     commits_since = check_output(f'git log {last_tag}..HEAD --oneline')
153 |     commit_messages = []
154 |     for commit in commits_since.splitlines():
155 |         wo_sha = re.sub('^[a-f0-9]{7} ', '', commit)
156 |         commit_messages.append(wo_sha)
157 | 
158 |     print(' NEW CHANGE LOG '.center(80, '='))
159 |     change_log = []
160 |     head = '{} ({})'.format(
161 |         next_version,
162 |         datetime.datetime.now().strftime('%Y-%m-%d')
163 |     )
164 |     head += '\n{}'.format('-' * len(head))
165 |     change_log.append(head)
166 |     change_log.extend(['- {}'.format(x) for x in commit_messages])
167 |     print('\n\n'.join(change_log))
168 |     print('=' * 80)
169 | 
170 |     assert commit_messages
171 | 
172 |     # Edit jobs/setup.py
173 |     with open('jobs/setup.py') as f:
174 |         setup_py = f.read()
175 | 
176 |     if "version='{}',".format(current_version) not in setup_py:
177 |         print(
178 |             f"PROBLEM. You have {current_version} installed in your "
179 |             f"environment but appears there is a more recent version listed "
180 |             f"in jobs/setup.py.\n"
181 |             f"Consider running:\n\n"
182 |             f"    cd jobs && pip install -e . && cd ..\n\n"
183 |             f"And try again."
184 |         )
185 |         return 4
186 |     setup_py = setup_py.replace(
187 |         "version='{}',".format(current_version),
188 |         "version='{}',".format(next_version),
189 |     )
190 |     if not dry_run:
191 |         with open('jobs/setup.py', 'w') as f:
192 |             f.write(setup_py)
193 | 
194 |     # Edit jobs/CHANGELOG.rst
195 |     with open('jobs/CHANGELOG.rst') as f:
196 |         original = f.read()
197 |     assert '\n\n'.join(change_log) not in original
198 |     new_change_log = original.replace(
199 |         '=========',
200 |         '=========\n\n{}\n\n'.format(
201 |             '\n\n'.join(change_log)
202 |         )
203 |     )
204 |     if not dry_run:
205 |         with open('jobs/CHANGELOG.rst', 'w') as f:
206 |             f.write(new_change_log)
207 | 
208 |     # Actually commit this change.
209 |     commit_message = f'Bump {next_version}'
210 |     if dry_run:
211 |         print('git add jobs/CHANGELOG.rst jobs/setup.py')
212 |         print(
213 |             f'git commit -m "{commit_message}"'
214 |         )
215 |     else:
216 |         subprocess.check_call([
217 |             'git', 'add', 'jobs/CHANGELOG.rst', 'jobs/setup.py',
218 |         ])
219 |         subprocess.check_call([
220 |             'git', 'commit', '-m', commit_message,
221 |         ])
222 | 
223 |     # Commit these changes
224 |     tag_name = tag_name_format.format(version=next_version)
225 |     tag_body = '\n\n'.join(['- {}'.format(x) for x in commit_messages])
226 |     if dry_run:
227 |         print(
228 |             f'git tag -s -a {tag_name} -m "...See CHANGELOG output above..."'
229 |         )
230 |     else:
231 |         subprocess.check_call([
232 |             'git',
233 |             'tag',
234 |             '-s',
235 |             '-a', tag_name,
236 |             '-m', tag_body,
237 |         ])
238 | 
239 |     # Let's push this now
240 |     if dry_run:
241 |         print(f'git push {upstream_name} master --tags')
242 |     else:
243 |         subprocess.check_call(
244 |             f'git push {upstream_name} master --tags'.split()
245 |         )
246 | 
247 |     if not dry_run:
248 |         release = _create_release(
249 |             github_api_key,
250 |             tag_name,
251 |             tag_body,
252 |             name=tag_name,
253 |         )
254 |         asset_info = _upload_lambda_zip(
255 |             github_api_key,
256 |             release['upload_url'],
257 |             release['id'],
258 |             f'buildhub-lambda-{tag_name}.zip',
259 |         )
260 |         print('Build asset uploaded.')
261 |         print('Can be downloaded at:')
262 |         print(asset_info['browser_download_url'])
263 | 
264 |         print('\n')
265 |         print(' 🎉  ALL DONE! 🎊 ')
266 |         print('\n')
267 | 
268 |     return 0
269 | 
270 | 
271 | def _create_release(github_api_key, tag_name, body, name=''):
272 |     api_url = (
273 |         f'https://api.github.com'
274 |         f'/repos/{OWNER}/{REPO}/releases'
275 |     )
276 |     response = requests.post(
277 |         api_url,
278 |         json={
279 |             'tag_name': tag_name,
280 |             'body': body,
281 |             'name': name,
282 |         }, headers={
283 |             'Authorization': 'token {}'.format(github_api_key)
284 |         }
285 |     )
286 |     response.raise_for_status()
287 |     return response.json()
288 | 
289 | 
290 | def _upload_lambda_zip(github_api_key, upload_url, release_id, filename):
291 |     upload_url = upload_url.replace(
292 |         '{?name,label}',
293 |         f'?name={filename}',
294 |     )
295 |     print('Uploading lambda.zip as {} ({})...'.format(
296 |         filename,
297 |         _format_file_size(os.stat('lambda.zip').st_size)
298 |     ))
299 |     with open('lambda.zip', 'rb') as f:
300 |         response = requests.get(
301 |             upload_url,
302 |             data=f,
303 |             headers={
304 |                 'Content-Type': 'application/zip',
305 |                 'Authorization': 'token {}'.format(github_api_key)
306 |             },
307 |         )
308 |     response.raise_for_status()
309 |     return response.json()
310 | 
311 | 
312 | if __name__ == '__main__':
313 |     import sys
314 |     import argparse
315 |     parser = argparse.ArgumentParser(description='Release Buildhub!')
316 |     parser.add_argument('part', type=str, help='major, minor or patch')
317 |     parser.add_argument('-d', '--dry-run', action='store_true')
318 |     parser.add_argument(
319 |         '-g', '--github-api-key',
320 |         help='GitHub API key unless set by GITHUB_API_KEY env var.'
321 |     )
322 |     parser.add_argument(
323 |         '-u', '--upstream-name',
324 |         help=(
325 |             'Name of the git remote origin to push to. Not your fork. '
326 |             'Defaults to "origin".'
327 |         ),
328 |         default='origin',
329 |     )
330 |     args = parser.parse_args()
331 |     if args.part not in ('major', 'minor', 'patch'):
332 |         parser.error("invalid part. Must be 'major', 'minor', or 'patch'")
333 |     args = vars(args)
334 |     sys.exit(main(args.pop('part'), **args))
335 | 


--------------------------------------------------------------------------------
/bin/run.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | set -eo pipefail
 3 | 
 4 | usage() {
 5 |   echo "usage: ./bin/run.sh version|waitfor|functional-tests|unit-tests|lintcheck|lambda.zip|initialize-kinto|latest-inventory-to-kinto"
 6 |   echo ""
 7 |   echo "    version                     Show current version"
 8 |   echo "    waitfor                     Run ./bin/wait-for"
 9 |   echo "    functional-tests            Run all tests"
10 |   echo "    unit-tests                  Run all tests except the functional ones"
11 |   echo "    lintcheck                   Run flake8 on all code"
12 |   echo "    lambda.zip                  Build Zip archive from buildhub package"
13 |   echo "    latest-inventory-to-kinto   Load latest S3 inventory to a Kinto server"
14 |   echo ""
15 |   exit 1
16 | }
17 | 
18 | [ $# -lt 1 ] && usage
19 | 
20 | case $1 in
21 |   version)
22 |     cat version.json
23 |     ;;
24 |   waitfor)
25 |     ./bin/wait-for ${@:2}
26 |     ;;
27 |   unit-tests)
28 |     py.test --ignore=jobs/tests/test_lambda_s3_event_functional.py --ignore=jobs/tests/test_lambda_s3_event_functional.py --override-ini="cache_dir=/tmp/tests" jobs/tests ${@:2}
29 |     ;;
30 |   functional-tests)
31 |     SERVER_URL=http://testkinto:9999/v1 py.test --override-ini="cache_dir=/tmp/tests" jobs/tests ${@:2}
32 |     ;;
33 |   lambda.zip)
34 |     rm -fr .venv
35 |     python -m venv .venv
36 |     source .venv/bin/activate
37 |     pip install -I ./jobs
38 |     pip install -r jobs/requirements/default.txt -c jobs/requirements/constraints.txt
39 |     pushd .venv/lib/python3.6/site-packages/
40 |     zip -r /app/lambda.zip *
41 |     popd
42 |     rm -fr .venv
43 |     ;;
44 |   latest-inventory-to-kinto)
45 |     latest-inventory-to-kinto ${@:2}
46 |     ;;
47 |   lintcheck)
48 |     flake8 jobs/buildhub jobs/tests
49 |     ;;
50 |   *)
51 |     exec "$@"
52 |     ;;
53 | esac
54 | 


--------------------------------------------------------------------------------
/bin/sanspreamble.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | # This Source Code Form is subject to the terms of the Mozilla Public
 4 | # License, v. 2.0. If a copy of the MPL was not distributed with this
 5 | # file, you can obtain one at http://mozilla.org/MPL/2.0/.
 6 | 
 7 | import os
 8 | import subprocess
 9 | import fnmatch
10 | 
11 | 
12 | def run():
13 | 
14 |     exceptions = (
15 |         '.*',
16 |         'docs/conf.py',
17 |         'setup.py',
18 |         'registerServiceWorker.js',
19 |     )
20 | 
21 |     def is_exception(fp):
22 |         if os.path.basename(fp) in exceptions:
23 |             return True
24 |         for exception in exceptions:
25 |             if fnmatch.fnmatch(fp, exception):
26 |                 return True
27 |         return False
28 | 
29 |     alreadies = subprocess.check_output([
30 |         'git', 'grep',
31 |         'This Source Code Form is subject to the terms of the Mozilla Public'
32 |     ]).decode('utf-8')
33 |     alreadies = [x.split(':')[0] for x in alreadies.splitlines()]
34 | 
35 |     out = subprocess.check_output(['git', 'ls-files']).decode('utf-8')
36 | 
37 |     suspect = []
38 |     for fp in out.splitlines():
39 |         if fp in alreadies:
40 |             continue
41 |         if not os.stat(fp).st_size:
42 |             continue
43 |         if is_exception(fp):
44 |             continue
45 |         if True in map(fp.endswith, ('.py', '.js')):
46 |             suspect.append(fp)
47 | 
48 |     for i, fp in enumerate(suspect):
49 |         if not i:
50 |             print('The following appear to lack a license preamble:'.upper())
51 |         print(fp)
52 | 
53 |     return len(suspect)
54 | 
55 | 
56 | if __name__ == '__main__':
57 |     import sys
58 |     sys.exit(run())
59 | 


--------------------------------------------------------------------------------
/bin/upload_to_s3.py:
--------------------------------------------------------------------------------
 1 | # This Source Code Form is subject to the terms of the Mozilla Public
 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this
 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/.
 4 | 
 5 | from __future__ import print_function
 6 | 
 7 | import boto3
 8 | import boto3.session
 9 | import os
10 | 
11 | AWS_REGION = "eu-central-1"
12 | BUCKET_NAME = "buildhub-lambda"
13 | 
14 | FILENAME = "lambda.zip"
15 | 
16 | s3 = boto3.resource('s3', region_name=AWS_REGION)
17 | try:
18 |     s3.create_bucket(Bucket=BUCKET_NAME)
19 | except:
20 |     pass
21 | 
22 | print('Uploading %s to Amazon S3 bucket %s' % (FILENAME, BUCKET_NAME))
23 | s3.Object(BUCKET_NAME, 'lambda.zip').put(Body=open(FILENAME, 'rb'))
24 | 
25 | print('File uploaded to https://s3.%s.amazonaws.com/%s/lambda.zip' %
26 |       (AWS_REGION, BUCKET_NAME))
27 | 


--------------------------------------------------------------------------------
/bin/wait-for:
--------------------------------------------------------------------------------
  1 | #!/bin/sh
  2 | 
  3 | # From https://github.com/Eficode/wait-for
  4 | 
  5 | # The MIT License (MIT)
  6 | #
  7 | # Copyright (c) 2017 Eficode Oy
  8 | #
  9 | # Permission is hereby granted, free of charge, to any person obtaining a copy
 10 | # of this software and associated documentation files (the "Software"), to deal
 11 | # in the Software without restriction, including without limitation the rights
 12 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 13 | # copies of the Software, and to permit persons to whom the Software is
 14 | # furnished to do so, subject to the following conditions:
 15 | #
 16 | # The above copyright notice and this permission notice shall be included in all
 17 | # copies or substantial portions of the Software.
 18 | #
 19 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 20 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 21 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 22 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 23 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 24 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 25 | # SOFTWARE.
 26 | 
 27 | 
 28 | 
 29 | TIMEOUT=15
 30 | QUIET=0
 31 | 
 32 | echoerr() {
 33 |   if [ "$QUIET" -ne 1 ]; then printf "%s\n" "$*" 1>&2; fi
 34 | }
 35 | 
 36 | usage() {
 37 |   exitcode="$1"
 38 |   cat << USAGE >&2
 39 | Usage:
 40 |   $cmdname host:port [-t timeout] [-- command args]
 41 |   -q | --quiet                        Do not output any status messages
 42 |   -t TIMEOUT | --timeout=timeout      Timeout in seconds, zero for no timeout
 43 |   -- COMMAND ARGS                     Execute command with args after the test finishes
 44 | USAGE
 45 |   exit "$exitcode"
 46 | }
 47 | 
 48 | wait_for() {
 49 |   for i in `seq $TIMEOUT` ; do
 50 |     nc -z "$HOST" "$PORT" > /dev/null 2>&1
 51 | 
 52 |     result=$?
 53 |     if [ $result -eq 0 ] ; then
 54 |       if [ $# -gt 0 ] ; then
 55 |         exec "$@"
 56 |       fi
 57 |       exit 0
 58 |     fi
 59 |     sleep 1
 60 |   done
 61 |   echo "Operation timed out" >&2
 62 |   exit 1
 63 | }
 64 | 
 65 | while [ $# -gt 0 ]
 66 | do
 67 |   case "$1" in
 68 |     *:* )
 69 |     HOST=$(printf "%s\n" "$1"| cut -d : -f 1)
 70 |     PORT=$(printf "%s\n" "$1"| cut -d : -f 2)
 71 |     shift 1
 72 |     ;;
 73 |     -q | --quiet)
 74 |     QUIET=1
 75 |     shift 1
 76 |     ;;
 77 |     -t)
 78 |     TIMEOUT="$2"
 79 |     if [ "$TIMEOUT" = "" ]; then break; fi
 80 |     shift 2
 81 |     ;;
 82 |     --timeout=*)
 83 |     TIMEOUT="${1#*=}"
 84 |     shift 1
 85 |     ;;
 86 |     --)
 87 |     shift
 88 |     break
 89 |     ;;
 90 |     --help)
 91 |     usage 0
 92 |     ;;
 93 |     *)
 94 |     echoerr "Unknown argument: $1"
 95 |     usage 1
 96 |     ;;
 97 |   esac
 98 | done
 99 | 
100 | if [ "$HOST" = "" -o "$PORT" = "" ]; then
101 |   echoerr "Error: you need to provide a host and port to test."
102 |   usage 2
103 | fi
104 | 
105 | wait_for "$@"
106 | 


--------------------------------------------------------------------------------
/docker-compose.ci.yml:
--------------------------------------------------------------------------------
 1 | # This file is specifically only to be used in CircleCI for docker-compose
 2 | # commads.
 3 | 
 4 | version: '2'
 5 | 
 6 | services:
 7 |   testkinto:
 8 |     build:
 9 |       # So it can read files from within 'jobs/'
10 |       context: .
11 |       dockerfile: testkinto/Dockerfile
12 |     ports:
13 |       - "9999:9999"
14 | 
15 |   buildhub:
16 |     build:
17 |       context: .
18 |       dockerfile: Dockerfile
19 |       args:
20 |         - CI
21 |     image: mozilla/buildhub
22 |     depends_on:
23 |       - "testkinto"
24 |     environment:
25 |       - SERVER_URL=http://testkinto:9999/v1
26 |     command: functional-tests
27 | 
28 |   docs:
29 |     build:
30 |       context: .
31 |       dockerfile: docs/Dockerfile
32 |       args:
33 |         - CI
34 |     command: build
35 | 
36 |   ui:
37 |     build:
38 |        context: .
39 |        dockerfile: ui/Dockerfile
40 |     environment:
41 |       - NODE_ENV=development
42 |     command: start
43 | 


--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
 1 | version: '2'
 2 | 
 3 | services:
 4 |   db:
 5 |     # As of Mar 8 2017, we run 9.6.6 in Buildhub production.
 6 |     image: postgres:9.6
 7 | 
 8 |   elasticsearch:
 9 |     # As of Mar 8 2017, we run es 5.4.0 in Buildhub production.
10 |     image: docker.elastic.co/elasticsearch/elasticsearch:5.4.0
11 |     environment:
12 |       - cluster.name=docker-cluster
13 |       - bootstrap.memory_lock=true
14 |       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
15 |       - xpack.security.enabled=false
16 |     ulimits:
17 |       memlock:
18 |         soft: -1
19 |         hard: -1
20 |     mem_limit: 1g
21 |     ports:
22 |       - 9200:9200
23 | 
24 |   kinto:
25 |     build:
26 |       # So it can read files from within 'jobs/'
27 |       context: .
28 |       dockerfile: kinto/Dockerfile
29 |     depends_on:
30 |       - "db"
31 |       - "elasticsearch"
32 |     ports:
33 |       - "8888:8888"
34 |     volumes:
35 |       - $PWD:/app
36 | 
37 |   testkinto:
38 |     build:
39 |       # So it can read files from within 'jobs/'
40 |       context: .
41 |       dockerfile: testkinto/Dockerfile
42 |     ports:
43 |       - "9999:9999"
44 |     # XXX Not sure we need this. What business do you have in there?!
45 |     volumes:
46 |       - $PWD:/app
47 | 
48 |   buildhub:
49 |     build:
50 |       context: .
51 |       dockerfile: Dockerfile
52 |     image: mozilla/buildhub
53 |     depends_on:
54 |       - "kinto"
55 |     environment:
56 |       - SERVER_URL=http://kinto:8888/v1
57 |       - CSV_DOWNLOAD_DIRECTORY=./csv-download-directory
58 |     volumes:
59 |       - $PWD:/app
60 |       - ~/.bash_history:/root/.bash_history
61 |     # All things within the 'buildhub' directive are meant to be executed
62 |     # with 'docker-compose run buildhub ...'
63 |     # By setting this to "true", it means we can run `docker-compose up`
64 |     # and this one won't start anything.
65 |     command: "true"
66 | 
67 |   ui:
68 |     build:
69 |        context: .
70 |        dockerfile: ui/Dockerfile
71 |     depends_on:
72 |       - "kinto"
73 |     environment:
74 |       - REACT_APP_KINTO_COLLECTION_URL=/v1/buckets/build-hub/collections/releases/
75 |       - NODE_ENV=development
76 |     ports:
77 |       - "3000:3000"
78 |       - "35729:35729"
79 |     volumes:
80 |       - $PWD/ui:/app
81 |     command: start
82 | 
83 |   docs:
84 |     build:
85 |       context: .
86 |       dockerfile: docs/Dockerfile
87 |     volumes:
88 |       - $PWD:/app
89 |     # Do nothing when 'docker-compose up'
90 |     command: "true"
91 | 


--------------------------------------------------------------------------------
/docs/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM python:3.6-slim
 2 | MAINTAINER Product Delivery irc://irc.mozilla.org/#storage-team
 3 | 
 4 | ENV PYTHONUNBUFFERED=1 \
 5 |     PYTHONPATH=/app/
 6 | 
 7 | # install a few essentials and clean apt caches afterwards
 8 | RUN apt-get update && \
 9 |     apt-get install -y --no-install-recommends \
10 |         apt-transport-https build-essential curl
11 | 
12 | # Clean up apt
13 | RUN apt-get autoremove -y && \
14 |     apt-get clean && \
15 |     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
16 | 
17 | COPY docs/requirements.txt /tmp/
18 | WORKDIR /tmp
19 | RUN pip install --no-cache-dir -r requirements.txt
20 | 
21 | COPY . /app
22 | 
23 | # Switch back to home directory
24 | WORKDIR /app
25 | 
26 | 
27 | # Using /bin/bash as the entrypoint works around some volume mount issues on Windows
28 | # where volume-mounted files do not have execute bits set.
29 | # https://github.com/docker/compose/issues/2301#issuecomment-154450785 has additional background.
30 | ENTRYPOINT ["/bin/bash", "/app/docs/run.sh"]
31 | 
32 | CMD ["build"]
33 | 


--------------------------------------------------------------------------------
/docs/Makefile:
--------------------------------------------------------------------------------
 1 | # Minimal makefile for Sphinx documentation
 2 | #
 3 | 
 4 | # You can set these variables from the command line.
 5 | SPHINXOPTS    = -W
 6 | SPHINXBUILD   = sphinx-build
 7 | SPHINXPROJ    = Buildhub
 8 | SOURCEDIR     = .
 9 | BUILDDIR      = _build
10 | 
11 | # Put it first so that "make" without argument is like "make help".
12 | help:
13 | 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
14 | 
15 | .PHONY: help Makefile
16 | 
17 | # Catch-all target: route all unknown targets to Sphinx using the new
18 | # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
19 | %: Makefile
20 | 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
21 | 


--------------------------------------------------------------------------------
/docs/_static/theme_overrides.css:
--------------------------------------------------------------------------------
 1 | #toc h3 {
 2 |     margin-bottom: 0.1em;
 3 | }
 4 | 
 5 | #toc {
 6 |     margin-top: 3em;
 7 |     display: flex;
 8 |     flex-wrap: wrap;
 9 | }
10 | 
11 | #toc .item {
12 |     padding-bottom: 1em;
13 |     padding-right: 0.5em;
14 | 
15 |     flex: 1 1 auto;
16 |     width: 230px;
17 | }
18 | 
19 | #toc .description {
20 |     margin: 0px;
21 |     padding: 0px;
22 |     margin-bottom: 0.5em;
23 | }
24 | 
25 | #toc .links {
26 |     margin: 0.1em;
27 |     padding: 0.1em;
28 |     font-size: 0.8em;
29 | }
30 | 
31 | /* override table width restrictions */
32 | .wy-table-responsive table td, .wy-table-responsive table th {
33 |     /* !important prevents the common CSS stylesheets from
34 |        overriding this as on RTD they are loaded after this stylesheet */
35 |     white-space: normal !important;
36 | }
37 | 
38 | .wy-table-responsive {
39 |     overflow: visible !important;
40 | }
41 | 
42 | /* small tweak for overview images */
43 | #overview table img {
44 |     margin-right: 30px;
45 |     float: left;
46 | }
47 | 
48 | #overview #key-features .wy-table-responsive:nth-of-type(1) table td {
49 |     font-size: 1.2em;
50 | }
51 | 
52 | /* tweak to limit width of column for settings default values */
53 | 
54 | .wy-table-responsive table tr td:nth-child(2), .wy-table-responsive tr table th:nth-child(2) {
55 |     white-space: -pre-wrap !important;
56 | }
57 | 
58 | #settings table tr td:nth-child(2) code {
59 |     max-width: 100px;
60 |     overflow-wrap: break-word;
61 |     word-break: break-word;
62 |     word-wrap: break-word;
63 |     white-space: pre-wrap;
64 | }
65 | 


--------------------------------------------------------------------------------
/docs/conf.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | #
  3 | # Configuration file for the Sphinx documentation builder.
  4 | #
  5 | # This file does only contain a selection of the most common options. For a
  6 | # full list see the documentation:
  7 | # http://www.sphinx-doc.org/en/master/config
  8 | 
  9 | # -- Path setup --------------------------------------------------------------
 10 | 
 11 | # If extensions (or modules to document with autodoc) are in another directory,
 12 | # add these directories to sys.path here. If the directory is relative to the
 13 | # documentation root, use os.path.abspath to make it absolute, like shown here.
 14 | #
 15 | # import os
 16 | # import sys
 17 | # sys.path.insert(0, os.path.abspath('.'))
 18 | 
 19 | 
 20 | # -- Project information -----------------------------------------------------
 21 | 
 22 | project = 'Buildhub'
 23 | copyright = '2018, Mozilla Product Delivery Team'
 24 | author = 'Mozilla Product Delivery Team'
 25 | 
 26 | # The short X.Y version
 27 | version = ''
 28 | # The full version, including alpha/beta/rc tags
 29 | release = ''
 30 | 
 31 | 
 32 | # -- General configuration ---------------------------------------------------
 33 | 
 34 | keep_warnings = True
 35 | 
 36 | # If your documentation needs a minimal Sphinx version, state it here.
 37 | #
 38 | # needs_sphinx = '1.0'
 39 | 
 40 | # Add any Sphinx extension module names here, as strings. They can be
 41 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 42 | # ones.
 43 | extensions = [
 44 |     'sphinx.ext.autodoc',
 45 |     'sphinxcontrib.httpdomain',
 46 |     'sphinx.ext.extlinks',
 47 |     'sphinx.ext.intersphinx',
 48 | ]
 49 | 
 50 | # Add any paths that contain templates here, relative to this directory.
 51 | templates_path = ['_templates']
 52 | 
 53 | # The suffix(es) of source filenames.
 54 | # You can specify multiple suffix as a list of string:
 55 | #
 56 | # source_suffix = ['.rst', '.md']
 57 | source_suffix = '.rst'
 58 | 
 59 | # The master toctree document.
 60 | master_doc = 'index'
 61 | 
 62 | # The language for content autogenerated by Sphinx. Refer to documentation
 63 | # for a list of supported languages.
 64 | #
 65 | # This is also used if you do content translation via gettext catalogs.
 66 | # Usually you set "language" from the command line for these cases.
 67 | language = None
 68 | 
 69 | # List of patterns, relative to source directory, that match files and
 70 | # directories to ignore when looking for source files.
 71 | # This pattern also affects html_static_path and html_extra_path .
 72 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
 73 | 
 74 | # The name of the Pygments (syntax highlighting) style to use.
 75 | pygments_style = 'sphinx'
 76 | 
 77 | 
 78 | def setup(app):
 79 |     # path relative to _static
 80 |     app.add_stylesheet('theme_overrides.css')
 81 | 
 82 | 
 83 | # -- Options for HTML output -------------------------------------------------
 84 | 
 85 | # The theme to use for HTML and HTML Help pages.  See the documentation for
 86 | # a list of builtin themes.
 87 | #
 88 | # html_theme = 'alabaster'
 89 | html_theme = 'sphinx_rtd_theme'
 90 | 
 91 | # Theme options are theme-specific and customize the look and feel of a theme
 92 | # further.  For a list of options available for each theme, see the
 93 | # documentation.
 94 | #
 95 | # html_theme_options = {}
 96 | 
 97 | # Add any paths that contain custom static files (such as style sheets) here,
 98 | # relative to this directory. They are copied after the builtin static files,
 99 | # so a file named "default.css" will overwrite the builtin "default.css".
100 | html_static_path = ['_static']
101 | 
102 | # Custom sidebar templates, must be a dictionary that maps document names
103 | # to template names.
104 | #
105 | # The default sidebars (for documents that don't match any pattern) are
106 | # defined by theme itself.  Builtin themes are using these templates by
107 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
108 | # 'searchbox.html']``.
109 | #
110 | # html_sidebars = {}
111 | 
112 | 
113 | # -- Options for HTMLHelp output ---------------------------------------------
114 | 
115 | # Output file base name for HTML help builder.
116 | htmlhelp_basename = 'Buildhubdoc'
117 | 
118 | 
119 | # -- Options for LaTeX output ------------------------------------------------
120 | 
121 | latex_elements = {
122 |     # The paper size ('letterpaper' or 'a4paper').
123 |     #
124 |     # 'papersize': 'letterpaper',
125 | 
126 |     # The font size ('10pt', '11pt' or '12pt').
127 |     #
128 |     # 'pointsize': '10pt',
129 | 
130 |     # Additional stuff for the LaTeX preamble.
131 |     #
132 |     # 'preamble': '',
133 | 
134 |     # Latex figure (float) alignment
135 |     #
136 |     # 'figure_align': 'htbp',
137 | }
138 | 
139 | # Grouping the document tree into LaTeX files. List of tuples
140 | # (source start file, target name, title,
141 | #  author, documentclass [howto, manual, or own class]).
142 | latex_documents = [
143 |     (master_doc, 'Buildhub.tex', 'Buildhub Documentation',
144 |      'Mozilla Product Delivery Team', 'manual'),
145 | ]
146 | 
147 | 
148 | # -- Options for manual page output ------------------------------------------
149 | 
150 | # One entry per manual page. List of tuples
151 | # (source start file, name, description, authors, manual section).
152 | man_pages = [
153 |     (master_doc, 'buildhub', 'Buildhub Documentation',
154 |      [author], 1)
155 | ]
156 | 
157 | 
158 | # -- Options for Texinfo output ----------------------------------------------
159 | 
160 | # Grouping the document tree into Texinfo files. List of tuples
161 | # (source start file, target name, title, author,
162 | #  dir menu entry, description, category)
163 | texinfo_documents = [
164 |     (master_doc, 'Buildhub', 'Buildhub Documentation',
165 |      author, 'Buildhub', 'One line description of project.',
166 |      'Miscellaneous'),
167 | ]
168 | 
169 | 
170 | # -- Options of extlinks --------------------------------------------------
171 | 
172 | extlinks = {
173 |     'github': ('https://github.com/%s/', ''),
174 |     'rtd': ('https://%s.readthedocs.io', ''),
175 |     'blog': ('http://www.servicedenuages.fr/%s', '')
176 | }
177 | 
178 | 
179 | # -- Options for autodoc --------------------------------------------------
180 | 
181 | autodoc_member_order = 'bysource'
182 | # Enable nitpicky mode - which ensures that all references in the docs
183 | # resolve.
184 | # See: http://stackoverflow.com/a/30624034/186202
185 | nitpicky = True
186 | nitpick_ignore = [
187 |     ('py:obj', 'Exception'),
188 |     ('py:obj', 'bool'),
189 |     ('py:obj', 'cornice.Service'),
190 |     ('py:obj', 'dict'),
191 |     ('py:obj', 'float'),
192 |     ('py:obj', 'int'),
193 |     ('py:obj', 'list'),
194 |     ('py:obj', 'str'),
195 |     ('py:obj', 'tuple'),
196 |     # Member autodoc fails with those:
197 |     # kinto.core.resource.schema
198 |     ('py:class', 'Integer'),
199 |     ('py:class', 'String'),
200 |     # kinto.core.resource
201 |     ('py:class', 'Model'),
202 |     ('py:class', 'ResourceSchema'),
203 |     ('py:class', 'ShareableModel'),
204 |     ('py:class', 'ShareableViewSet'),
205 |     ('py:class', 'ViewSet'),
206 |     ('py:class', 'Sequence')
207 | ]
208 | 


--------------------------------------------------------------------------------
/docs/configuration.rst:
--------------------------------------------------------------------------------
  1 | .. _configuration:
  2 | 
  3 | Configuration
  4 | #############
  5 | 
  6 | 
  7 | You can set the following environment variables:
  8 | 
  9 | 
 10 | Cron job AND Lambda
 11 | ===================
 12 | 
 13 | ``SENTRY_DSN``
 14 | --------------
 15 | **Type:** String
 16 | 
 17 | **Default:** ``None``
 18 | 
 19 | Use sending crashes to http://sentry.prod.mozaws.net/
 20 | 
 21 | 
 22 | ``SERVER_URL``
 23 | --------------
 24 | **Type:** String
 25 | 
 26 | **Default:** ``http://localhost:8888/v1``
 27 | 
 28 | URL to the Kinto server for storage.
 29 | 
 30 | 
 31 | ``BUCKET``
 32 | ----------
 33 | **Type:** String
 34 | 
 35 | **Default:** ``build-hub``
 36 | 
 37 | Name of Kinto bucket to store it in.
 38 | 
 39 | 
 40 | ``COLLECTION``
 41 | --------------
 42 | **Type:** String
 43 | 
 44 | **Default:** ``releases``
 45 | 
 46 | Name of Kinto collection to store it in.
 47 | 
 48 | 
 49 | ``AUTH``
 50 | --------
 51 | **Type:** String
 52 | 
 53 | **Default:** ``user:pass``
 54 | 
 55 | Credentials for connecting to the Kinto server.
 56 | 
 57 | 
 58 | ``TIMEOUT_SECONDS``
 59 | -------------------
 60 | **Type:** Integer
 61 | 
 62 | **Default:** 300
 63 | 
 64 | Max. seconds for fetching JSON URLs.
 65 | 
 66 | 
 67 | ``NB_PARALLEL_REQUESTS``
 68 | ------------------------
 69 | **Type:** Integer
 70 | 
 71 | **Default:** 8
 72 | 
 73 | Used both for number of parallel HTTP GET requests and the number of records
 74 | to send in each batch operation to Kinto.
 75 | 
 76 | 
 77 | ``PRODUCTS``
 78 | ------------
 79 | **Type:** List of strings separated by space
 80 | 
 81 | **Default:** ``firefox thunderbird mobile devedition``
 82 | 
 83 | Which product names to not ignore.
 84 | 
 85 | ``ARCHIVE_URL``
 86 | ---------------
 87 | **Type:** String
 88 | 
 89 | **Default:** ``https://archive.mozilla.org/``
 90 | 
 91 | Primarily is an environment variable so it can be changed with when running
 92 | unit tests to check that the unit tests actually don't depend on real
 93 | network calls.
 94 | 
 95 | ``LOG_METRICS``
 96 | ---------------
 97 | **Type:** String
 98 | 
 99 | **Default:** ``datadog``
100 | 
101 | Used to decided which `backend for markus to use <http://markus.readthedocs.io/en/latest/backends.html>`_.
102 | Value must be one of ``datadog``, ``logging``, ``void`` or ``cloudwatch``.
103 | 
104 | If set to ``datadog`` (which is default), it will then send ``statsd`` commands
105 | to ``${STATSD_HOST}:${STATSD_PORT}`` (with namespace ``${STATSD_NAMESPACE}``).
106 | Their defaults are: ``localhost``, ``8125`` and ``''``.
107 | 
108 | If set to ``logging``, instead of sending metrics command anywhere, they are
109 | included in the configured Python logging with prefix of ``metrics``.
110 | 
111 | If set to ``void``, all metrics are ignored and not displayed or sent anywhere.
112 | 
113 | If set to ``cloudwatch``, prints to ``stdout`` a custom format used by
114 | `Datadog and AWS Cloudswatch <https://docs.datadoghq.com/integrations/amazon_lambda/>`_
115 | that AWS Lambda can use.
116 | 
117 | Cron job ONLY
118 | =============
119 | 
120 | ``INITIALIZE_SERVER``
121 | ---------------------
122 | **Type:** Bool
123 | 
124 | **Default:** True
125 | 
126 | Whether to initialize the Kinto server on first run.
127 | 
128 | 
129 | ``MIN_AGE_LAST_MODIFIED_HOURS``
130 | -------------------------------
131 | **Type:** Integer
132 | 
133 | **Default:** 0  (disabled)
134 | 
135 | When processing the CSV files, how "young" does an entry have to be to be
136 | considered at all. Entries older than this are skipped.
137 | 
138 | 
139 | ``CSV_DOWNLOAD_DIRECTORY``
140 | --------------------------
141 | 
142 | **Type:** String
143 | 
144 | **Default:** ``/tmp`` (as per Python's ``tempfile.gettempdir()``)
145 | 
146 | Path to where CSV files, from the inventory, is stored. When using Docker
147 | for local development, it's advantageous to set this to a mounted directory.
148 | E.g. ``./csv-downloads`` so that the data isn't lost between runs of
149 | stopping and starting the Docker container.
150 | 
151 | 
152 | ``INVENTORIES``
153 | ---------------
154 | **Type:** List of strings
155 | 
156 | **Default:** ``firefox, archive``
157 | 
158 | Which inventories to scrape and in which order.
159 | 
160 | 
161 | Lamba ONLY
162 | ==========
163 | 
164 | ``NB_RETRY_REQUEST``
165 | --------------------
166 | **Type:** Integer
167 | 
168 | **Default:** 6
169 | 
170 | Specifically sent to the ``backoff`` function for how many (exponentially
171 | increasing sleeps) attempts to try to re-read if a HTTP GET fails.
172 | 
173 | 
174 | ``CACHE_FOLDER``
175 | ----------------
176 | **Type:** String
177 | 
178 | **Default:** ``.``
179 | 
180 | Before the scraping really starts, we build up a Python dict of every
181 | previously saved release ID and it's hash (as a string from the ``data``).
182 | This is used to be able to quickly answer *"Did we already have this release
183 | and has it not changed?"*.
184 | 
185 | Once this has been built up, that Python dict is dumped to disk, in
186 | ``CACHE_FOLDER``, so that next time it's faster to generate this dict without
187 | having to query the whole Kinto database.
188 | 


--------------------------------------------------------------------------------
/docs/dev.rst:
--------------------------------------------------------------------------------
  1 | .. _dev:
  2 | 
  3 | Developer Documentation
  4 | #######################
  5 | 
  6 | To do development locally all you need is ``docker`` and ``docker-compose``.
  7 | 
  8 | Quick Start
  9 | ===========
 10 | 
 11 | To start everything just run:
 12 | 
 13 | .. code-block:: shell
 14 | 
 15 |     $ docker-compose up
 16 | 
 17 | This will start:
 18 | 
 19 | * A ``kinto`` server backed by PostgreSQL that syncs everything to
 20 |   Elasticsearch (accessible via ``localhost:8888``)
 21 | * A PostgreSQL server
 22 | * An Elasticsearch server (accessible via ``localhost:9200``)
 23 | * A ``kinto`` memory-storage server for running functional tests against
 24 |   (accessible via ``localhost:9999``)
 25 | * A (``create-react-app``) React server (accessible via ``localhost:3000``)
 26 | 
 27 | The very first time you run it, the database will be empty. You need to
 28 | populate it with an initial "scrape". More about that in the following
 29 | section.
 30 | 
 31 | 
 32 | Local development server
 33 | ========================
 34 | 
 35 | First start the ``kinto`` server:
 36 | 
 37 | .. code-block:: shell
 38 | 
 39 |     $ docker-compose up kinto
 40 | 
 41 | This will start a ``kinto`` server you can reach via ``localhost:8888``.
 42 | It needs to be bootstrapped once. To that run (in another terminal):
 43 | 
 44 | .. code-block:: shell
 45 | 
 46 |     $ docker-compose run kinto migrate
 47 |     $ docker-compose run kinto initialize-kinto-wizard jobs/buildhub/initialization.yml  --server http://kinto:8888/v1 --auth user:pass
 48 | 
 49 | You should now have a running PostgreSQL and Elasticsearch that you can
 50 | populate with buildhub data and inspect. To see what's inside the PostgreSQL
 51 | server you can run:
 52 | 
 53 | .. code-block:: shell
 54 | 
 55 |     $ docker-compose run db psql -h db -U postgres
 56 | 
 57 | To see what's in the Elasticsearch you can simply open
 58 | ``http://localhost:9200`` in your browser.
 59 | 
 60 | 
 61 | Initial data
 62 | ============
 63 | 
 64 | The above steps will set up a working but blank ``kinto`` database. I.e.
 65 | no actual records in PostgreSQL and no documents indexed in Elasticsearch.
 66 | 
 67 | To boostrap the data, you can run ``latest-inventory-to-kinto``. You do that
 68 | like this:
 69 | 
 70 | 
 71 | .. code-block:: shell
 72 | 
 73 |     $ docker-compose run buildhub bash
 74 |     app@b95573edb130:~$ latest-inventory-to-kinto
 75 | 
 76 | That'll take a while. A really long while. Be prepared to go for lunch and
 77 | leave your computer on.
 78 | 
 79 | .. note::
 80 | 
 81 |     Don't worry about it being called "LATEST-inventory-to-kinto". It just
 82 |     means it uses the latest *manifest* (generated once a day) but it
 83 |     still iterates over every single build in S3.
 84 | 
 85 | 
 86 | Incremental data
 87 | ================
 88 | 
 89 | When you've run ``latest-inventory-to-kinto`` at least once, the second time
 90 | it will be faster because, before sending data to ``kinto`` the
 91 | ``to_kinto.py`` script will generate a massive dictionary of all previously
 92 | inserted IDs. However, all the "scraping" still needs to be processed again.
 93 | If you want to you can set an environment variable first that will
 94 | quickly ignore any S3 objects that are "too old". For example:
 95 | 
 96 | .. code-block:: shell
 97 | 
 98 |     $ docker-compose run buildhub bash
 99 |     app@b95573edb130:~$ MIN_AGE_LAST_MODIFIED_HOURS=48 latest-inventory-to-kinto
100 | 
101 | With ``MIN_AGE_LAST_MODIFIED_HOURS=48`` it means that any build that is found
102 | to be older than 2 days from today (in UTC) will immediately bit skipped.
103 | This will drastically reduce the time it takes to run the whole job.
104 | 
105 | Functional tests
106 | ================
107 | 
108 | The functional tests require that you have a ``kinto`` server up and running.
109 | By default, the functional tests assumes that the ``kinto`` server is running
110 | at ``http://testkinto:9999``. To start that server, run:
111 | 
112 | .. code-block:: shell
113 | 
114 |     $ docker-compose up testkinto
115 | 
116 | 
117 | .. note::
118 | 
119 |     The reason there are **two** ``kinto`` servers (one for functional tests
120 |     and one for a local dev instance) is because the functional tests have
121 |     fixture expectations and can't guarantee that it leaves the database in
122 |     the same state as *before* the tests.
123 |     It would be annoying if you local instance changes weirdly (potentially
124 |     unrealistic names) every you run the funtional tests.
125 | 
126 | From the host you can test that it's running with ``curl``:
127 | 
128 | .. code-block:: shell
129 | 
130 |     $ curl localhost:9999
131 | 
132 | At least once, prime the ``kinto`` server like this:
133 | 
134 | .. code-block:: shell
135 | 
136 |     $ docker-compose run kinto initialize-kinto-wizard jobs/buildhub/initialization.yml  --server http://testkinto:9999/v1 --auth user:pass
137 | 
138 | To start the functional tests now, run:
139 | 
140 | .. code-block:: shell
141 | 
142 |     $ docker-compose run buildhub functional-tests
143 | 
144 | Note that the default ``docker-compose.yml`` sets up a volume mount. So
145 | if you change any of the files in the current directory, it's immediately
146 | used in the next ``docker-compose run ...`` run.
147 | 
148 | .. note::
149 | 
150 |     In the instructions above, you had to have two terminals open. One for the
151 |     ``kinto`` server and one for the running of the tests. Alternatively
152 |     you can use ``docker-compose up -d testkinto`` to put it in the background.
153 |     Use ``docker-compose ps`` to see that it's running.
154 |     And when you no longer need it, instead of ``Ctrl-C`` in that terminal,
155 |     run ``docker-compose stop``. Again, use ``docker-compose ps`` to see that
156 |     it's *not* running.
157 | 
158 | Unit tests
159 | ==========
160 | 
161 | The unit tests are basically the functional tests except the tests
162 | that depend on the presence of an actual server available. These tests
163 | are faster to run when iterating rapidly when you know the
164 | functional test isn't immediately important or relevant.
165 | 
166 | Simple run:
167 | 
168 | .. code-block:: shell
169 | 
170 |     $ docker-compose run buildhub unit-tests
171 | 
172 | Unlike the functional tests, this does not require first running
173 | ``docker-compose up kinto``.
174 | 
175 | 
176 | Lint checking
177 | =============
178 | 
179 | To ``flake8`` check all the tests and jobs code run:
180 | 
181 | .. code-block:: shell
182 | 
183 |     $ docker-compose run buildhub lintcheck
184 | 
185 | 
186 | Generating ``lambda.zip`` file
187 | ==============================
188 | 
189 | The ``lambda.zip`` is a zip file for the ``site-packages`` of a Python 3.6
190 | that has ``buildhub`` and all its dependencies including the ``.pyc`` files.
191 | 
192 | To generate the file use:
193 | 
194 | .. code-block:: shell
195 | 
196 |     $ docker-compose run buildhub lambda.zip
197 | 
198 | .. note::
199 | 
200 |     You might want to assert that the ``buildhub`` ``.pyc`` files really
201 |     are there. ``unzip -l lambda.zip | grep buildhub``.
202 | 
203 | Running pytests like a pro
204 | ==========================
205 | 
206 | If you're hacking on something and find that typing
207 | ``docker-compose run buildhub functional-tests`` is too slow, there is a
208 | better way to run ``pytest`` more iteratively and more rapidly. To do
209 | so enter a ``bash`` shell as ``root``:
210 | 
211 | .. code-block:: shell
212 | 
213 |     $ docker-compose run --user 0 buildhub bash
214 | 
215 | From here you can run ``pytest`` with all its possible options.
216 | For example:
217 | 
218 | .. code-block:: shell
219 | 
220 |     $ export SERVER_URL=http://testkinto:9999/v1
221 |     $ pytest jobs/tests/ -x --ff --showlocals --tb=native
222 | 
223 | .. note::
224 | 
225 |     The ``SERVER_URL`` has to be set to point to the ``testkinto``
226 |     server if you're going to run functional tests from within this
227 |     bash session.
228 | 
229 | And you can now install ``pytest-watch`` to have the tests run as soon
230 | as you save any of the relevant files:
231 | 
232 | 
233 | .. code-block:: shell
234 | 
235 |     $ pip install pytest-watch
236 |     # Now replace `pytest` with `ptw -- `
237 |     $ ptw -- jobs/tests/ -x --ff --showlocals --tb=native
238 | 
239 | Writing documentation
240 | =====================
241 | 
242 | To work on the documentation, edit the ``docs/**/*.rst`` files. To build
243 | and see the result of your work run:
244 | 
245 | .. code-block:: shell
246 | 
247 |     $ docker-compose run docs build
248 | 
249 | That will generate the ``docs/_build/html/`` directory (unless there were
250 | errors) and you can open the ``index.html`` in your browser:
251 | 
252 | .. code-block:: shell
253 | 
254 |     $ open docs/_build/html/index.html
255 | 
256 | .. note::
257 | 
258 |     Our ``sphinx-build`` command transforms any warnings into errors.
259 | 
260 | Upgrading UI packages
261 | =====================
262 | 
263 | To update any of the packages that the UI uses, follow these steps
264 | (as an example):
265 | 
266 | .. code-block:: shell
267 | 
268 |     $ docker-compose run ui bash
269 |     root@be2dda49e5ef:/app# yarn outdated
270 |     root@be2dda49e5ef:/app# yarn upgrade prettier --latest
271 |     root@be2dda49e5ef:/app# exit
272 | 
273 | Check that the ``ui/package.json`` and ``ui/yarn.lock`` changed accordingly.
274 | 
275 | Licensing preamble
276 | ==================
277 | 
278 | Every file of code we write, since we use Mozilla Public License 2.0, has to
279 | have a preamble. The best way to describe is to look at existing code.
280 | There's a script to check if any files have been missed:
281 | 
282 | .. code-block:: shell
283 | 
284 |     $ ./bin/sanspreamble.py
285 | 
286 | Run it to check which files you might have missed.
287 | 
288 | Adding/Updating Python packages to requirements files
289 | =====================================================
290 | 
291 | You don't need a virtualenv on the host as long as you have
292 | `hashin <https://pypi.python.org/pypi/hashin>`_ installed globally.
293 | 
294 | Alternatively you can use Docker. In this example we add a new package
295 | called ``markus``:
296 | 
297 | .. code-block:: shell
298 | 
299 |     $ docker-compose run --user 0 buildhub bash
300 |     root@2ec530633121:/app# pip install hashin
301 |     root@2ec530633121:/app# cd jobs/
302 |     root@2ec530633121:/app/jobs# hashin markus -r requirements/default.txt
303 | 
304 | Some packages might require additonal "contraint packages". Best way to
305 | know is to find out by seeing if ``pip install ...`` fails:
306 | 
307 | 
308 | .. code-block:: shell
309 | 
310 |     root@2ec530633121:/app/jobs# pip install -r requirements/default.txt -c requirements/constraints.txt
311 | 
312 | If it fails because an extra constraint package is required add it like this:
313 | 
314 | .. code-block:: shell
315 | 
316 |     root@2ec530633121:/app/jobs# hashin some-extra-package -r requirements/constraints.txt
317 | 
318 | Finally, exit the interactive Docker shell and try to rebuild:
319 | 
320 | .. code-block:: shell
321 | 
322 |     $ docker-compose build buildhub
323 | 
324 | Debugging metrics logging
325 | =========================
326 | 
327 | By default, all metrics logging goes to
328 | ``markus.backends.datadog.DatadogMetrics``. This requires that you have a
329 | ``statsd`` server running on ``$STATSD_HOST:STATSD_PORT``. If you don't
330 | have that locally you can change it to plain Python logging by setting
331 | ``LOG_METRICS=logging``. For example:
332 | 
333 | .. code-block:: shell
334 | 
335 |     $ docker-compose run buildhub bash
336 |     app@b95573edb130:~$ LOG_METRICS=logging latest-inventory-to-kinto
337 | 
338 | The current only allowed values for the ``LOG_METRICS`` environment variable
339 | is ``datadog`` (default) and ``logging``. Anything else will raise a
340 | ``NotImplementedError`` exception.
341 | 
342 | .. note::
343 | 
344 |     Another option if you want *no metrics output at all* is to set
345 |     ``LOG_METRICS=void`` and then all metrics logging is swallowed and
346 |     ignored.
347 | 
348 | Environment Variables and Settings files
349 | ========================================
350 | 
351 | Every environment variable that is defined and expected can obviously be
352 | changed on the command line. E.g.
353 | 
354 | .. code-block:: shell
355 | 
356 |     app@b95573edb130:~$ CSV_DOWNLOAD_DIRECTORY=/tmp/stuff latest-inventory-to-kinto
357 | 
358 | But you can also, put all *your* preferences in a ``.env`` file or a
359 | ``settings.ini`` file in the root of the project. These are yours to keep and
360 | not check in.
361 | 
362 | The order of preference is as follows:
363 | 
364 | 1. Command line
365 | 2. ``settings.ini`` file
366 | 3. ``.env`` file
367 | 
368 | See documentation on `python-decouple here
369 | <https://pypi.org/project/python-decouple/#id5>`_.
370 | 
371 | The rule of thumb is that all environment variables defined in the Python
372 | code should **be for the purpose of production run**. Meaning, if you want
373 | something that is not how we run it, by default, in production it's up to you
374 | to override it.
375 | 
376 | Also, this rule of thumb implies that environment variables should be
377 | secure by default. If there's a sensitive setting, don't leave insecure for
378 | developers and expecting OPs to make it explicitly secure. It's the other
379 | way around. If you want to make some less secure (perhaps it's fine because
380 | you're running it on your laptop with fake data), it's up to you to either
381 | specify it on the command line or update your personal ``.env`` file.
382 | 
383 | 
384 | Network Leak Detection in Unit Tests
385 | ====================================
386 | 
387 | Some tests are unit tests where the network is attempted to be mocked. These
388 | tests should never depend on a working network connection. The functional
389 | tests though (files like ``test_something_functional.py``) depend on actually
390 | using the network.
391 | 
392 | To test that the unit tests don't accidentally depend on a URL that hasn't
393 | been mocked, attempt to run the unit test exclusively but before you do that
394 | set ``ARCHIVE_URL`` to something that would never work. E.g.
395 | 
396 | .. code-block:: shell
397 | 
398 |     $ docker-compose run buildhub bash
399 |     app@b95573edb130:~$ ARCHIVE_URL=https://archive.example.xom/ py.test --ignore=jobs/tests/test_inventory_to_records_functional.py --ignore=jobs/tests/test_lambda_s3_event_functional.py --override-ini="cache_dir=/tmp/tests" jobs/tests
400 | 


--------------------------------------------------------------------------------
/docs/index.rst:
--------------------------------------------------------------------------------
 1 | Overview
 2 | ########
 3 | 
 4 | 
 5 | *Buildhub* aims to provide a public database of comprehensive information about `Mozilla products <https://www.mozilla.org>`_ releases and builds.
 6 | 
 7 | Concretely, it is a JSON API (`Kinto <https://kinto-storage.org>`_) where you can query a collection of records.
 8 | 
 9 | Quickstart
10 | ==========
11 | 
12 | Browse the database
13 | -------------------
14 | 
15 | * `Online catalog <https://mozilla-services.github.io/buildhub/>`_
16 | 
17 | Basic JSON API
18 | --------------
19 | 
20 | Buildhub is just a collection of records on a Kinto server.
21 | 
22 | * `$SERVER/buckets/build-hub/collections/releases/records?_limit=10 <https://buildhub.prod.mozaws.net/v1/buckets/build-hub/collections/releases/records?_limit=10>`_
23 | 
24 | A set of filters and pagination options can be used to query the collection. See :ref:`the dedicated section <api>`.
25 | 
26 | Elasticsearch API
27 | -----------------
28 | 
29 | An ElasticSearch endpoint is also available for more powerful queries:
30 | 
31 | * `$SERVER/buckets/build-hub/collections/releases/search <https://buildhub.prod.mozaws.net/v1/buckets/build-hub/collections/releases/search>`_
32 | 
33 | `More information in the Elasticsearch documentation <https://www.elastic.co/guide/en/elasticsearch/reference/current/search.html>`_
34 | 
35 | Data License
36 | ------------
37 | 
38 | Since the information published on *Buildhub* is compiled from publicly available sources, the data is published in the `Public Domain <https://en.wikipedia.org/wiki/Public-domain_software>`_.
39 | 
40 | 
41 | Table of contents
42 | =================
43 | 
44 | .. toctree::
45 |    :maxdepth: 2
46 | 
47 |    api
48 |    jobs
49 |    support
50 |    dev
51 |    configuration
52 | 


--------------------------------------------------------------------------------
/docs/jobs.rst:
--------------------------------------------------------------------------------
  1 | .. _jobs:
  2 | 
  3 | Jobs
  4 | ####
  5 | 
  6 | A script will aggregate all build information from Mozilla archives, and another is in charge of keeping it up to date.
  7 | 
  8 | Everything can be executed from a command-line, but we use `Amazon Lambda <https://aws.amazon.com/lambda/>`_ in production.
  9 | 
 10 | .. image:: overview.png
 11 | 
 12 | Currently we use `Kinto <http://kinto-storage.org>`_ as a generic database service. It allows us to leverage its simple API for storing and querying records. It also comes with a set of client libraries for JavaScript, Python etc.
 13 | 
 14 | 
 15 | Initialization
 16 | ==============
 17 | 
 18 | .. note::
 19 | 
 20 |     The ``user:pass`` in the command-line examples is the Basic auth for Kinto.
 21 | 
 22 | The following is not mandatory but recommended. Kinto can use the JSON schema to validate the records. The following setting should be set to ``true`` in the server configuration file:
 23 | 
 24 | .. code-block:: ini
 25 | 
 26 |     kinto.experimental_collection_schema_validation = true
 27 | 
 28 | 
 29 | Load latest S3 inventory
 30 | ========================
 31 | 
 32 | A command to initialize the remote Kinto server, download the latest S3 manifests, containing information about all available files on archive.mozilla.org, and send that information as buildhub records to the remote Kinto server.
 33 | 
 34 | .. code-block:: bash
 35 | 
 36 |     latest-inventory-to-kinto
 37 | 
 38 | The command will go through the list of files, pick release files, and deduce their metadata. It is meant to be executed on an empty server, or periodically to catch up with recent releases in case the other event-based lambda had failed.
 39 | 
 40 | Its configuration is read from environment variables:
 41 | 
 42 | * ``SERVER_URL`` (default: ``http://localhost:8888/v1``)
 43 | * ``BUCKET`` (default: ``build-hub``)
 44 | * ``COLLECTION`` (default: ``releases``)
 45 | * ``AUTH`` (default: ``user:pass``)
 46 | * ``CACHE_FOLDER`` (default: ``.``)
 47 | * ``NB_RETRY_REQUEST`` (default: ``3``)
 48 | * ``BATCH_MAX_REQUESTS`` (default: taken from server)
 49 | * ``TIMEOUT_SECONDS`` (default: ``300``)
 50 | * ``INITIALIZE_SERVER`` (default: ``true``): whether to initialize the destination bucket/collection.
 51 | * ``SENTRY_DSN`` (default: empty/disabled. Example: ``https://<key>:<secret>@sentry.io/buildhub``)
 52 | * ``MIN_AGE_LAST_MODIFIED_HOURS`` (default: ``0`` which disables it): number of days of age to consider analyzing and comparing against database.
 53 | 
 54 | S3 Event lambda
 55 | ===============
 56 | 
 57 | The Amazon Lambda function that is in charge of keeping the database up-to-date. This one cannot be executed from the command-line.
 58 | 
 59 | When releases are published on S3, an `S3 Event <http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html>`_ is triggered and `the lambda is invoked <http://docs.aws.amazon.com/lambda/latest/dg/with-s3.html>`_.
 60 | 
 61 | Use the following entry point:
 62 | 
 63 | * ``buildhub.lambda_s3_event.lambda_handler``
 64 | 
 65 | .. note::
 66 | 
 67 |     Since release records contain information from JSON metadata files, we handle the case when the JSON metdata file is published before the actual archive, and vice-versa.
 68 | 
 69 | The lambda accepts the following configuration (from environment variables):
 70 | 
 71 | * ``SERVER_URL`` (default: ``http://localhost:8888/v1``)
 72 | * ``BUCKET`` (default: ``build-hub``)
 73 | * ``COLLECTION`` (default: ``releases``)
 74 | * ``CACHE_FOLDER`` (default: ``.``)
 75 | * ``AUTH`` (default: ``user:pass``)
 76 | * ``NB_RETRY_REQUEST`` (default: ``3``)
 77 | * ``TIMEOUT_SECONDS`` (default: ``300``)
 78 | * ``SENTRY_DSN`` (default: empty/disabled. Example: ``https://<key>:<secret>@sentry.io/buildhub``)
 79 | 
 80 | 
 81 | Setup and configure Amazon Lambda
 82 | =================================
 83 | 
 84 | In order to build the AWS Lambda Zip archive in an isolated environment, we use Docker:
 85 | 
 86 | * ``make lambda.zip``
 87 | 
 88 | (...or most likely ``sudo make lambda.zip``)
 89 | 
 90 | This will produce a zip file that has to be uploaded in AWS Lambda configuration panel.
 91 | 
 92 | .. image:: lambda-1.png
 93 | .. image:: lambda-2.png
 94 | .. image:: lambda-3.png
 95 | .. image:: lambda-4.png
 96 | 
 97 | 
 98 | Using Docker
 99 | ============
100 | 
101 | Some commands are exposed in the container entry-point command (``docker run``).
102 | 
103 | The exhaustive list of available commands and description is available using:
104 | 
105 | ::
106 | 
107 |     docker run -t mozilla/buildhub
108 | 
109 | For example, run tests:
110 | 
111 | ::
112 | 
113 |     docker run -t mozilla/buildhub test
114 | 
115 | Or load the latest S3 inventory:
116 | 
117 | ::
118 | 
119 |     docker run -e "SERVER_URL=https://buildhub.prod.mozaws.net/v1" -e "AUTH=user:pass" -t mozilla/buildhub latest-inventory-to-kinto
120 | 
121 | 
122 | Load S3 inventory manually
123 | ==========================
124 | 
125 | In order to fetch inventories from S3, install the dedicated Amazon Services client:
126 | 
127 | .. code-block:: bash
128 | 
129 |    sudo apt-get install awscli
130 | 
131 | We are interested in two listing: ``firefox`` and ``archive`` (thunderbird, mobile).
132 | 
133 | .. code-block:: bash
134 | 
135 |     export LISTING=archive
136 | 
137 | List available manifests in the inventories folder:
138 | 
139 | .. code-block:: bash
140 | 
141 |     aws --no-sign-request --region us-east-1 s3 ls "s3://net-mozaws-prod-delivery-inventory-us-east-1/public/inventories/net-mozaws-prod-delivery-$LISTING/delivery-$LISTING/"
142 | 
143 | Download the latest manifest:
144 | 
145 | .. code-block:: bash
146 | 
147 |     aws --no-sign-request --region us-east-1 s3 cp s3://net-mozaws-prod-delivery-inventory-us-east-1/public/inventories/net-mozaws-prod-delivery-$LISTING/delivery-$LISTING/2017-08-02T00-11Z/manifest.json
148 | 
149 | Download the associated files (using `jq <https://stedolan.github.io/jq/download/>`_):
150 | 
151 | .. code-block:: bash
152 | 
153 |     files=$(jq -r '.files[] | .key' < 2017-08-01T00-12Z/manifest.json)
154 |     for file in $files; do
155 |         aws --no-sign-request --region us-east-1 s3 cp "s3://net-mozaws-prod-delivery-inventory-us-east-1/public/$file" .
156 |     done
157 | 
158 | Initialize the remote server from a manifest that will define the buckets, collection, records schema, and related permissions. This command is idempotent, and will only modify existing objects if something was changed.
159 | 
160 | .. code-block:: bash
161 | 
162 |     kinto-wizard load --server https://kinto/ --auth user:pass jobs/buildhub/initialization.yml
163 | 
164 | Parse S3 inventory, fetch metadata, and print records as JSON in stdout:
165 | 
166 | .. code-block:: bash
167 | 
168 |     zcat *.csv.gz | inventory-to-records > records.data
169 | 
170 | Load records into Kinto:
171 | 
172 | .. code-block:: bash
173 | 
174 |     cat records.data | to-kinto --server https://kinto/ --bucket build-hub --collection release --auth user:pass
175 | 
176 | Repeat with ``LISTING=firefox``.
177 | 
178 | .. note::
179 | 
180 |     All three commands can be piped together with their respective parameters::
181 | 
182 |         zcat *.csv.gz | inventory-to-records | to-kinto
183 | 


--------------------------------------------------------------------------------
/docs/lambda-1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mozilla-services/buildhub/82919da2d31bd645948cb4e8281c0d8c83c707a3/docs/lambda-1.png


--------------------------------------------------------------------------------
/docs/lambda-2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mozilla-services/buildhub/82919da2d31bd645948cb4e8281c0d8c83c707a3/docs/lambda-2.png


--------------------------------------------------------------------------------
/docs/lambda-3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mozilla-services/buildhub/82919da2d31bd645948cb4e8281c0d8c83c707a3/docs/lambda-3.png


--------------------------------------------------------------------------------
/docs/lambda-4.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mozilla-services/buildhub/82919da2d31bd645948cb4e8281c0d8c83c707a3/docs/lambda-4.png


--------------------------------------------------------------------------------
/docs/overview.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mozilla-services/buildhub/82919da2d31bd645948cb4e8281c0d8c83c707a3/docs/overview.png


--------------------------------------------------------------------------------
/docs/requirements.txt:
--------------------------------------------------------------------------------
1 | sphinx
2 | sphinx-autobuild
3 | sphinx_rtd_theme
4 | sphinxcontrib-httpdomain
5 | 


--------------------------------------------------------------------------------
/docs/run.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | set -eo pipefail
 3 | 
 4 | usage() {
 5 |   echo "usage: ./run.sh build"
 6 |   echo ""
 7 |   echo "    build                         Generate the docs/_build/html/ files"
 8 |   echo ""
 9 |   exit 1
10 | }
11 | 
12 | 
13 | [ $# -lt 1 ] && usage
14 | 
15 | case $1 in
16 |   build)
17 |     cd docs && make html
18 |     ;;
19 |   *)
20 |     exec "$@"
21 |     ;;
22 | esac
23 | 


--------------------------------------------------------------------------------
/docs/support.rst:
--------------------------------------------------------------------------------
 1 | .. _support:
 2 | 
 3 | Support
 4 | #######
 5 | 
 6 | Do not hesitate to ask questions, start conversations, or report issues on the `BuildHub Github repo <https://github.com/mozilla-services/buildhub/>`_.
 7 | 
 8 | .. note::
 9 | 
10 |     Currently we use the same issue tracker for problems related to code (eg. features, bugs) and data (eg. missing builds records) for every component (UI, server, lambda, cron job...).
11 | 
12 | 
13 | Frequently Asked Questions
14 | ==========================
15 | 
16 | Builds XYZ are missing, is it normal?
17 | -------------------------------------
18 | 
19 | It's never normal. Every missing data should be considered as bug and reported.
20 | 
21 | We have a cron job that backfills the data when a bug is fixed.
22 | 
23 | 
24 | How can I help you triage/debug?
25 | --------------------------------
26 | 
27 | Answering those questions may help:
28 | 
29 | - Is data missing for every product/version/platform/channel?
30 | - How old is the newest entry?
31 | - Does the missing release file on https://archive.mozilla.org have a different path/URL than the previous releases?
32 | 


--------------------------------------------------------------------------------
/jobs/CHANGELOG.rst:
--------------------------------------------------------------------------------
  1 | Changelog
  2 | =========
  3 | 
  4 | 1.4.1 (2019-06-10)
  5 | ------------------
  6 | 
  7 | - Merge pull request #535 from willkg/release-fix
  8 | 
  9 | - Fix release process
 10 | 
 11 | - Merge pull request #534 from willkg/deprecation-notice
 12 | 
 13 | - Fix lint issue
 14 | 
 15 | - Add deprecation notice
 16 | 
 17 | - Merge pull request #533 from willkg/update-deps
 18 | 
 19 | - Update js dependencies
 20 | 
 21 | - Update dependencies and fix building
 22 | 
 23 | - Update makefile
 24 | 
 25 | - Merge pull request #531 from willkg/coc
 26 | 
 27 | - Add code of conduct
 28 | 
 29 | - upgrade urllib3 (#529)
 30 | 
 31 | 
 32 | 
 33 | 1.4.0 (2018-11-16)
 34 | ------------------
 35 | 
 36 | - fixes bug 1506862 - linux/mac nightlies missing since nov 5th (#527)
 37 | 
 38 | - fix for inventory-to-kinto download_csv (#526)
 39 | 
 40 | 
 41 | 
 42 | 1.3.5 (2018-10-25)
 43 | ------------------
 44 | 
 45 | - upgrade requests to 2.20.0 (#524)
 46 | 
 47 | - the correct way to calm down pyup (#523)
 48 | 
 49 | - Disable pyup
 50 | 
 51 | - Merge pull request #492 from mozilla-services/pyup-update-backoff-1.5.0-to-1.6.0
 52 | 
 53 | - Update backoff from 1.5.0 to 1.6.0
 54 | 
 55 | - Update datadog to 0.22.0 (#489)
 56 | 
 57 | - adding new make run command
 58 | 
 59 | 
 60 | 
 61 | 1.3.4 (2018-05-18)
 62 | ------------------
 63 | 
 64 | - Lambda misses builds with spaces, fixes #468 (#469)
 65 | 
 66 | 
 67 | 
 68 | 1.3.3 (2018-05-15)
 69 | ------------------
 70 | 
 71 | - it's lambda not lamdba (#467)
 72 | 
 73 | - backoff even longer (#466)
 74 | 
 75 | - Update raven to 6.8.0 (#463)
 76 | 
 77 | - web ui should use stage kinto, fixes #458 (#460)
 78 | 
 79 | - Update backoff from 1.4.3 to 1.5.0 (#454)
 80 | 
 81 | - Update raven to 6.7.0 (#455)
 82 | 
 83 | - Update pytest from 3.5.0 to 3.5.1 (#426)
 84 | 
 85 | - Update aioresponses from 0.4.0 to 0.4.1 (#399)
 86 | 
 87 | - Update kinto-http from 9.1.1 to 9.1.2 (#420)
 88 | 
 89 | - update to deployment-bug now that stage auto-upgrades Lambda
 90 | 
 91 | 
 92 | 
 93 | 1.3.2 (2018-05-07)
 94 | ------------------
 95 | 
 96 | - insert buildhub release into raven config (#452)
 97 | 
 98 | - more avoid JSONFileNotFound in lambda event, fixes #424 (#453)
 99 | 
100 | 
101 | 
102 | 1.3.1 (2018-05-03)
103 | ------------------
104 | 
105 | - fetch_listing by locale in lambda should retry on 404, fixes #424 (#449)
106 | 
107 | - small fix for make-release script
108 | 
109 | 
110 | 
111 | 1.3.0 (2018-05-03)
112 | ------------------
113 | 
114 | - optimize fetch_existing, fixes #447 #445 #446 (#448)
115 | 
116 | - in lambda, fetch_listing with retry_on_notfound, fixes #424 (#442)
117 | 
118 | - Use ciso8601 to parse CSV dates, fixes #443 (#444)
119 | 
120 | - Fix readthedocs fixes 438 (#440)
121 | 
122 | - add link to Datadog dashboard
123 | 
124 | - fix for bad check_output in script
125 | 
126 | 
127 | 
128 | 1.2.1 (2018-04-27)
129 | ------------------
130 | 
131 | - fix for subprocess
132 | 
133 | - document all configurations (#436)
134 | 
135 | - Use cloudwatch metrics for lambda 433 (#435)
136 | 
137 | - use decouple for env vars to be able to read from .env
138 | 
139 | - more default NB_RETRY_REQUEST, fixes #425 (#432)
140 | 
141 | - Tests shouldn't depend on the network, fixes #429 (#430)
142 | 
143 | - Skip based on date earlier, fixes #427 (#428)
144 | 
145 | - Make release script (#422)
146 | 
147 | 
148 | 
149 | 1.2.0 (2018-04-23)
150 | ------------------
151 | 
152 | - run fetch_existing only once, fixes #412 (#414)
153 | 
154 | - fix sorting in searchKit (#419)
155 | 
156 | - add psql makefile target (#418)
157 | 
158 | - docs typo (#416)
159 | 
160 | - upgrade pytest to 3.5.0 (#415)
161 | 
162 | - Ability to quiet markus, fixes 410 (#411)
163 | 
164 | - Dump file is growing unsustainably fixes 394 (#404)
165 | 
166 | - bail on 404 responses in cron job (#389)
167 | 
168 | - use python-decouple everywhere (#409)
169 | 
170 | - Disk cache the manifests, fixes #392 (#403)
171 | 
172 | - Optionally configure markus to log fixes 402 (#405)
173 | 
174 | - pluggy sha's changed since they added wheels (#408)
175 | 
176 | - Update markus to 1.1.2 (#398)
177 | 
178 | - do raven for lambda correctly, fixes #387 (#391)
179 | 
180 | - Update .pyup.yml (#393)
181 | 
182 | - add a .pyup.yml file (#390)
183 | 
184 | - Add statsd metrics (#388)
185 | 
186 | - 80 lines pep8 (#384)
187 | 
188 | - refactor script name, fixes #377 (#383)
189 | 
190 | - skip old csv entries, fixes #380 (#382)
191 | 
192 | - drop css-animation, fixes #371 (#376)
193 | 
194 | - Use dockerhub repo (#375)
195 | 
196 | - change license to mpl 2.0, fixes #366 (#374)
197 | 
198 | - remove leftover console warn (#373)
199 | 
200 | - upgrade Prettier, fixes #369 (#372)
201 | 
202 | - bundle searchkit css, fixes #368 (#370)
203 | 
204 | - use yarn instead, fixes #365 (#367)
205 | 
206 | - Feature: RefinementAutosuggest for perf (#304)
207 | 
208 | - DOCKER_* env var names correction (#364)
209 | 
210 | - circle only (#358)
211 | 
212 | - pin all requirements, fixes #353 (#356)
213 | 
214 | 
215 | 1.1.5 (2018-02-22)
216 | ------------------
217 | 
218 | **Bug fixes**
219 | 
220 | - Now pick Windows .exe archives only (fixes #338)
221 | 
222 | 
223 | 1.1.4 (2018-02-15)
224 | ------------------
225 | 
226 | **Bug fixes**
227 | 
228 | - Be more robust about skipping non-date folders when looking for
229 |   manifests (ref https://bugzilla.mozilla.org/show_bug.cgi?id=1437931)
230 | - Retry requests on ``409 Conflict``
231 | 
232 | 
233 | 1.1.3 (2018-02-02)
234 | ------------------
235 | 
236 | - Retry fetch JSON when status is not 200 (ref #327)
237 | 
238 | **Bug fixes**
239 | 
240 | - Fix ordering of release candidates build folders (fixes #328)
241 | 
242 | **UI**
243 | 
244 | - Use classic ISO format for publication date (fixes #320)
245 | - Improve search placeholder (fixes #305)
246 | - Better favicon (fixes #306)
247 | - Add contribute.json endpoint (fixes #324)
248 | - Add link to Kinto record (fixes #286)
249 | 
250 | 
251 | 1.1.2 (2017-12-20)
252 | ------------------
253 | 
254 | - Fix event handling of RC metadata (fixes #314)
255 | - Fix exclusion of thunderbird nightly releases (fixes #312)
256 | - Prevent mozinfo JSON files to be mistaken as Nightly metadata (fixes #315)
257 | 
258 | 1.1.1 (2017-11-30)
259 | ------------------
260 | 
261 | - Fix test_packages regexp to avoid confusion with build metadata (fixes #295, #309)
262 | 
263 | 1.1.0 (2017-11-03)
264 | ------------------
265 | 
266 | - Changed log level from error to warning when metadata could not be found (#297, #298)
267 | - Updated docs with prod URLs (#293)
268 | - Added ElasticSearch queries examples (#294)
269 | 
270 | **Bug fixes**
271 | 
272 | - Use ``requirements.txt`` versions when building the container (fixes #299)
273 | - Prevent test_packages metadata from being recognized as release metadata (fixes #295)
274 | 
275 | 
276 | 1.0.0 (2017-10-12)
277 | ------------------
278 | 
279 | - Add ability to configure cache folder via environment variable ``CACHE_FOLDER``
280 | - Keep trace but skip build urls that have unsupported formats
281 | - Fix support of some funnelcake archives (fixes #287)
282 | - Skip very old RC release with parenthesis in filenames (fixes #288)
283 | 
284 | 
285 | 0.6.0 (2017-10-10)
286 | ------------------
287 | 
288 | - Add support for SNS events (#281)
289 | 
290 | 
291 | 0.5.0 (2017-10-10)
292 | ------------------
293 | 
294 | - Skip incomplete records ­- ie. without build id
295 | - Fix Mac OS X metadata URLs (fixes #261)
296 | - Fix Mac and Windows metadata URLs from installers (fixes #269)
297 | - Fix beta and devedition medata URLs (#269)
298 | - Skip exe installers where version is missing from URL (fixes #263)
299 | - Fix Fennec metadata location (fixes #264)
300 | - Fix caching when partial updates metadata is missing (fixes #276)
301 | - Fix handling of bad server response, like XML (fixes #259)
302 | 
303 | 
304 | 0.4.1 (2017-09-29)
305 | ------------------
306 | 
307 | - Fix S3 event ``eventTime`` key error (fixes #253)
308 | 
309 | 
310 | 0.4.0 (2017-09-14)
311 | ------------------
312 | 
313 | - Allow number of requests in batch to be overriden via environment variable ``BATCH_MAX_REQUESTS``.
314 | - Allow to run some commands from the container (fixes #41)
315 | 
316 | 0.3.0 (2017-09-06)
317 | ------------------
318 | 
319 | - Load ``initialization.yml`` from the S3 inventory lambda (#236)
320 | - Distinguish records cache files from a server to another (#235)
321 | - Major documentation improvements (#228)
322 | 
323 | 0.2.0 (2017-08-25)
324 | ------------------
325 | 
326 | - Add devedition to supported products. (#218)
327 | - Document S3 inventories lambda configuration. (#217)
328 | - Increase Gzip chunk size (#221)
329 | - Fix S3 manifest key (#220)
330 | - Add more build metadata (#219)
331 | - Fix Gzip decompressor (#225 / #227)
332 | - Skip WinCE and WinMo (#226)
333 | - Handle eabi-arm platform (#230)
334 | 
335 | 
336 | 0.1.0 (2017-08-18)
337 | ------------------
338 | 
339 | **Initial version**
340 | 
341 | - Read build information from S3 inventories and https://archives.mozilla.org
342 | - Lambda function to listen to S3 event
343 | - Lambda function to populate kinto from the S3 inventories.
344 | 


--------------------------------------------------------------------------------
/jobs/CONTRIBUTORS.rst:
--------------------------------------------------------------------------------
1 | Contributors
2 | ============
3 | 
4 | * Mathieu Agopian <magopian@mozilla.com>
5 | * Ethan Glasser-Camp <eglassercamp@mozilla.com>
6 | * Rémy Hubscher <rhubscher@mozilla.com>
7 | * Mathieu Leplatre <mathieu@mozilla.com>
8 | * Nicolas Perriault <nperriault@mozilla.com>
9 | 


--------------------------------------------------------------------------------
/jobs/MANIFEST.in:
--------------------------------------------------------------------------------
1 | include *.rst LICENSE Makefile tox.ini
2 | include buildhub/initialization.yml
3 | 


--------------------------------------------------------------------------------
/jobs/README.rst:
--------------------------------------------------------------------------------
1 | `Jobs documentation <https://buildhub.readthedocs.io/en/latest/jobs.html>`_


--------------------------------------------------------------------------------
/jobs/buildhub/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mozilla-services/buildhub/82919da2d31bd645948cb4e8281c0d8c83c707a3/jobs/buildhub/__init__.py


--------------------------------------------------------------------------------
/jobs/buildhub/configure_markus.py:
--------------------------------------------------------------------------------
  1 | # This Source Code Form is subject to the terms of the Mozilla Public
  2 | # License, v. 2.0. If a copy of the MPL was not distributed with this
  3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/.
  4 | import os
  5 | import time
  6 | 
  7 | import markus
  8 | from markus.backends import BackendBase
  9 | from decouple import config
 10 | 
 11 | 
 12 | _configured = False
 13 | 
 14 | 
 15 | def get_metrics(namespace):
 16 |     global _configured
 17 |     if not _configured:
 18 |         STATSD_HOST = config('STATSD_HOST', 'localhost')
 19 |         STATSD_PORT = config('STATSD_PORT', default=8125)
 20 |         STATSD_NAMESPACE = config('STATSD_NAMESPACE', default='')
 21 | 
 22 |         FILE_METRICS_BASE_DIR = config(
 23 |             'MARKUS_FILE_METRICS_BASE_DIR',
 24 |             default='/tmp'
 25 |         )
 26 | 
 27 |         # For more options see
 28 |         # http://markus.readthedocs.io/en/latest/usage.html#markus-configure
 29 |         log_metrics_config = config('LOG_METRICS', default='datadog')
 30 |         if log_metrics_config == 'logging':
 31 |             markus.configure([
 32 |                 {
 33 |                     'class': 'markus.backends.logging.LoggingMetrics',
 34 |                     'options': {
 35 |                         'logger_name': 'metrics'
 36 |                     }
 37 |                 }
 38 |             ])
 39 |         elif log_metrics_config == 'cloudwatch':
 40 |             markus.configure([
 41 |                 {
 42 |                     'class': 'markus.backends.cloudwatch.CloudwatchMetrics',
 43 |                 }
 44 |             ])
 45 |         elif log_metrics_config == 'datadog':
 46 |             markus.configure([
 47 |                 {
 48 |                     'class': 'markus.backends.datadog.DatadogMetrics',
 49 |                     'options': {
 50 |                         'statsd_host': STATSD_HOST,
 51 |                         'statsd_port': STATSD_PORT,
 52 |                         'statsd_namespace': STATSD_NAMESPACE,
 53 |                     }
 54 |                 }
 55 |             ])
 56 |         elif log_metrics_config == 'void':
 57 |             markus.configure([
 58 |                 {
 59 |                     'class': 'buildhub.configure_markus.VoidMetrics',
 60 |                 }
 61 |             ])
 62 |         elif log_metrics_config == 'file':
 63 |             markus.configure([
 64 |                 {
 65 |                     'class': 'buildhub.configure_markus.FileMetrics',
 66 |                     'options': {
 67 |                         'base_dir': FILE_METRICS_BASE_DIR,
 68 |                     }
 69 |                 }
 70 |             ])
 71 |         else:
 72 |             raise NotImplementedError(
 73 |                 f'Unrecognized LOG_METRICS value {log_metrics_config}'
 74 |             )
 75 |         _configured = True
 76 | 
 77 |     return markus.get_metrics(namespace)
 78 | 
 79 | 
 80 | class VoidMetrics(BackendBase):
 81 |     """Use when you want nothing with the markus metrics. E.g.
 82 | 
 83 |         markus.configure([
 84 |             {
 85 |                 'class': 'buildhub.configure_markus.VoidMetrics',
 86 |             }
 87 |         ])
 88 |     """
 89 | 
 90 |     def incr(self, stat, value, tags=None):
 91 |         pass
 92 | 
 93 |     def gauge(self, stat, value, tags=None):
 94 |         pass
 95 | 
 96 |     def timing(self, stat, value, tags=None):
 97 |         pass
 98 | 
 99 |     def histogram(self, stat, value, tags=None):
100 |         pass
101 | 
102 | 
103 | class FileMetrics(BackendBase):
104 |     """Use when you want to write the metrics to files.
105 | 
106 |         markus.configure([
107 |             {
108 |                 'class': 'buildhub.configure_markus.FileMetrics',
109 |                 'options': {
110 |                     'base_dir': '/my/log/path'
111 |                 }
112 |             }
113 |         ])
114 |     """
115 | 
116 |     def __init__(self, options):
117 |         self.prefix = options.get("prefix", "")
118 |         self.base_dir = options.get("base_dir", os.path.abspath("."))
119 |         self.fns = set()
120 |         os.makedirs(self.base_dir, exist_ok=True)
121 | 
122 |     def _log(self, metrics_kind, stat, value, tags):
123 |         tags = ("#%s" % ",".join(tags)) if tags else ""
124 |         fn = os.path.join(self.base_dir, "{}.{}.log".format(stat, metrics_kind))
125 |         with open(fn, "a") as f:
126 |             print("{:.3f}\t{}{}".format(time.time(), value, tags), file=f)
127 |         if fn not in self.fns:
128 |             print("Wrote first-time metrics in {}".format(fn))
129 |             self.fns.add(fn)
130 | 
131 |     def incr(self, stat, value=1, tags=None):
132 |         """Increment a counter"""
133 |         self._log("count", stat, value, tags)
134 | 
135 |     def gauge(self, stat, value, tags=None):
136 |         """Set a gauge"""
137 |         self._log("gauge", stat, value, tags)
138 | 
139 |     def timing(self, stat, value, tags=None):
140 |         """Set a timing"""
141 |         self._log("timing", stat, value, tags)
142 | 
143 |     def histogram(self, stat, value, tags=None):
144 |         """Set a histogram"""
145 |         self._log("histogram", stat, value, tags)
146 | 


--------------------------------------------------------------------------------
/jobs/buildhub/initialization.yml:
--------------------------------------------------------------------------------
  1 | build-hub:
  2 |   permissions:
  3 |     read:
  4 |       - system.Everyone
  5 |   collections:
  6 |     releases:
  7 |       data:
  8 |         index:schema:
  9 |           properties:
 10 |             id:
 11 |               type: "keyword"
 12 |               index: "not_analyzed"
 13 |             last_modified:
 14 |               type: "long"
 15 |             build:
 16 |               properties:
 17 |                 id:
 18 |                   type: "keyword"
 19 |                 date:
 20 |                   type: "date"
 21 |                   format: "date_time_no_millis"
 22 |             source:
 23 |               properties:
 24 |                 product:
 25 |                   type: "keyword"
 26 |                   index: "not_analyzed"
 27 |                 repository:
 28 |                   type: "keyword"
 29 |                   index: "not_analyzed"
 30 |                 tree:
 31 |                   type: "keyword"
 32 |                   index: "not_analyzed"
 33 |                 revision:
 34 |                   type: "keyword"
 35 |             target:
 36 |               properties:
 37 |                 platform:
 38 |                   type: "keyword"
 39 |                   index: "not_analyzed"
 40 |                 os:
 41 |                   type: "keyword"
 42 |                   index: "not_analyzed"
 43 |                 locale:
 44 |                   type: "keyword"
 45 |                   index: "not_analyzed"
 46 |                 version:
 47 |                   type: "keyword"
 48 |                   index: "not_analyzed"
 49 |                 channel:
 50 |                   type: "keyword"
 51 |                   index: "not_analyzed"
 52 |             download:
 53 |               properties:
 54 |                 url:
 55 |                   type: "keyword"
 56 |                 mimetype:
 57 |                   type: "keyword"
 58 |                   index: "not_analyzed"
 59 |                 size:
 60 |                   type: "long"
 61 |                 date:
 62 |                   type: "date"
 63 |                   format: "date_time_no_millis"
 64 |         displayFields:
 65 |           - id
 66 |           - source.product
 67 |           - target.channel
 68 |           - target.version
 69 |           - target.platform
 70 |           - target.locale
 71 |         schema:
 72 |           title: Release
 73 |           description: Mozilla software releases.
 74 |           type: object
 75 |           additionalProperties: false
 76 |           required:
 77 |             - source
 78 |             - download
 79 |             - target
 80 |           properties:
 81 |             build:
 82 |               type: object
 83 |               additionalProperties: false
 84 |               properties:
 85 |                 id:
 86 |                   type: string
 87 |                   title: Build ID
 88 |                   description: Build ID
 89 |                 date:
 90 |                   type: string
 91 |                   format: date-time
 92 |                   title: Build date
 93 |                   description: 'i.e: 2017-04-13T21:49:00Z'
 94 |                 number:
 95 |                   type: integer
 96 |                   title: Version
 97 |                   description: Build number
 98 |                 as:
 99 |                   type: string
100 |                   title: Assembler
101 |                   description: Executable
102 |                 ld:
103 |                   type: string
104 |                   title: Linker
105 |                   description: Executable
106 |                 cc:
107 |                   type: string
108 |                   title: C compiler
109 |                   description: Command-line
110 |                 cxx:
111 |                   type: string
112 |                   title: C++ compiler
113 |                   description: Command-line
114 |                 host:
115 |                   type: string
116 |                   title: Compiler host alias
117 |                   description: (cpu)-(vendor)-(os)
118 |                 target:
119 |                   type: string
120 |                   title: Target host alias
121 |                   description: (cpu)-(vendor)-(os)
122 |             target:
123 |               type: object
124 |               additionalProperties: false
125 |               required:
126 |                 - platform
127 |                 - locale
128 |                 - version
129 |                 - channel
130 |               properties:
131 |                 platform:
132 |                   type: string
133 |                   title: Platform
134 |                   description: Operating system and Architecture
135 |                 os:
136 |                   type: string
137 |                   title: OS
138 |                   description: Operating system family
139 |                   enum:
140 |                     - linux
141 |                     - win
142 |                     - mac
143 |                     - android
144 |                     - maemo
145 |                 locale:
146 |                   type: string
147 |                   title: Locale
148 |                 version:
149 |                   type: string
150 |                   title: Version
151 |                 channel:
152 |                   type: string
153 |                   title: Update channel
154 |             source:
155 |               type: object
156 |               additionalProperties: false
157 |               required:
158 |                 - product
159 |               properties:
160 |                 product:
161 |                   type: string
162 |                   title: Product
163 |                   description: Product name
164 |                 repository:
165 |                   type: string
166 |                   title: Repository
167 |                 tree:
168 |                   type: string
169 |                   title: Tree
170 |                   description: i.e mozilla-central
171 |                 revision:
172 |                   type: string
173 |                   title: Revision number in the tree
174 |             download:
175 |               type: object
176 |               additionalProperties: false
177 |               required:
178 |                 - url
179 |                 - mimetype
180 |                 - size
181 |                 - date
182 |               properties:
183 |                 url:
184 |                   type: string
185 |                   title: URL
186 |                   description: URL of the build
187 |                 mimetype:
188 |                   type: string
189 |                   title: Mimetype
190 |                 date:
191 |                   type: string
192 |                   format: date-time
193 |                   title: Date
194 |                   description: Build publication date
195 |                 size:
196 |                   type: integer
197 |                   title: Size
198 |                   description: In bytes
199 |         uiSchema:
200 |           ui:order:
201 |             - source
202 |             - download
203 |             - build
204 |             - target
205 |           download:
206 |             ui:order:
207 |               - url
208 |               - size
209 |               - date
210 |               - mimetype
211 |           source:
212 |             ui:order:
213 |               - product
214 |               - repository
215 |               - tree
216 |               - revision
217 |           build:
218 |             ui:order:
219 |               - id
220 |               - date
221 |               - number
222 |           target:
223 |             ui:order:
224 |               - platform
225 |               - os
226 |               - locale
227 |               - version
228 |               - channel
229 | 


--------------------------------------------------------------------------------
/jobs/buildhub/lambda_s3_event.py:
--------------------------------------------------------------------------------
  1 | # This Source Code Form is subject to the terms of the Mozilla Public
  2 | # License, v. 2.0. If a copy of the MPL was not distributed with this
  3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/.
  4 | 
  5 | import asyncio
  6 | import json
  7 | import logging
  8 | import re
  9 | import sys
 10 | 
 11 | import aiohttp
 12 | import ciso8601
 13 | import kinto_http
 14 | from decouple import config
 15 | from raven.contrib.awslambda import LambdaClient
 16 | 
 17 | from buildhub import utils
 18 | from buildhub.inventory_to_records import (
 19 |     __version__,
 20 |     NB_RETRY_REQUEST,
 21 |     fetch_json,
 22 |     fetch_listing,
 23 |     fetch_metadata,
 24 |     scan_candidates
 25 | )
 26 | from buildhub.configure_markus import get_metrics
 27 | 
 28 | 
 29 | # Optional Sentry with synchronuous client.
 30 | SENTRY_DSN = config('SENTRY_DSN', default=None)
 31 | sentry = LambdaClient(
 32 |     SENTRY_DSN,
 33 |     release=__version__,
 34 | )
 35 | 
 36 | logger = logging.getLogger()  # root logger.
 37 | metrics = get_metrics('buildhub')
 38 | 
 39 | 
 40 | async def main(loop, event):
 41 |     """
 42 |     Trigger when S3 event kicks in.
 43 |     http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html
 44 |     """
 45 |     server_url = config('SERVER_URL', default='http://localhost:8888/v1')
 46 |     bucket = config('BUCKET', default='build-hub')
 47 |     collection = config('COLLECTION', default='releases')
 48 |     kinto_auth = tuple(config('AUTH', 'user:pass').split(':'))
 49 | 
 50 |     kinto_client = kinto_http.Client(server_url=server_url, auth=kinto_auth,
 51 |                                      retry=NB_RETRY_REQUEST)
 52 | 
 53 |     records = []
 54 |     for record in event['Records']:
 55 |         if record.get('EventSource') == 'aws:sns':
 56 |             records.extend(json.loads(record['Sns']['Message'])['Records'])
 57 |         else:
 58 |             records.append(record)
 59 | 
 60 |     async with aiohttp.ClientSession(loop=loop) as session:
 61 |         for event_record in records:
 62 |             metrics.incr('s3_event_event')
 63 |             records_to_create = []
 64 | 
 65 |             # Use event time as archive publication.
 66 |             event_time = ciso8601.parse_datetime(event_record['eventTime'])
 67 |             event_time = event_time.strftime(utils.DATETIME_FORMAT)
 68 | 
 69 |             key = event_record['s3']['object']['key']
 70 | 
 71 |             filesize = event_record['s3']['object']['size']
 72 |             url = utils.key_to_archive_url(key)
 73 | 
 74 |             logger.debug("Event file {}".format(url))
 75 | 
 76 |             try:
 77 |                 product = key.split('/')[1]  # /pub/thunderbird/nightly/...
 78 |             except IndexError:
 79 |                 continue  # e.g. https://archive.mozilla.org/favicon.ico
 80 | 
 81 |             if product not in utils.ALL_PRODUCTS:
 82 |                 logger.info('Skip product {}'.format(product))
 83 |                 continue
 84 | 
 85 |             # Release / Nightly / RC archive.
 86 |             if utils.is_build_url(product, url):
 87 |                 logger.info('Processing {} archive: {}'.format(product, key))
 88 | 
 89 |                 record = utils.record_from_url(url)
 90 |                 # Use S3 event infos for the archive.
 91 |                 record['download']['size'] = filesize
 92 |                 record['download']['date'] = event_time
 93 | 
 94 |                 # Fetch release metadata.
 95 |                 await scan_candidates(session, product)
 96 |                 logger.debug("Fetch record metadata")
 97 |                 # metadata = await fetch_metadata(session, record)
 98 |                 metadata = await fetch_metadata(session, record)
 99 |                 # If JSON metadata not available, archive will be
100 |                 # handled when JSON is delivered.
101 |                 if metadata is None:
102 |                     logger.info(
103 |                         f"JSON metadata not available {record['id']}"
104 |                     )
105 |                     continue
106 | 
107 |                 # Merge obtained metadata.
108 |                 record = utils.merge_metadata(record, metadata)
109 |                 records_to_create.append(record)
110 | 
111 |             # RC metadata
112 |             elif utils.is_rc_build_metadata(product, url):
113 |                 logger.info(f'Processing {product} RC metadata: {key}')
114 | 
115 |                 # pub/firefox/candidates/55.0b12-candidates/build1/mac/en-US/
116 |                 # firefox-55.0b12.json
117 |                 logger.debug("Fetch new metadata")
118 |                 # It has been known to happen that right after an S3 Event
119 |                 # there's a slight delay to the metadata json file being
120 |                 # available. If that's the case we want to retry in a couple
121 |                 # of seconds to see if it's available on the next backoff
122 |                 # attempt.
123 |                 metadata = await fetch_json(
124 |                     session,
125 |                     url,
126 |                     retry_on_notfound=True
127 |                 )
128 |                 metadata['buildnumber'] = int(
129 |                     re.search('/build(\d+)/', url).group(1)
130 |                 )
131 | 
132 |                 # We just received the metadata file. Lookup if the associated
133 |                 # archives are here too.
134 |                 archives = []
135 |                 if 'multi' in url:
136 |                     # For multi we just check the associated archive
137 |                     # is here already.
138 |                     parent_folder = re.sub('multi/.+$', 'multi/', url)
139 |                     _, files = await fetch_listing(
140 |                         session,
141 |                         parent_folder,
142 |                         retry_on_notfound=True
143 |                     )
144 |                     for f in files:
145 |                         rc_url = parent_folder + f['name']
146 |                         if utils.is_build_url(product, rc_url):
147 |                             archives.append((
148 |                                 rc_url,
149 |                                 f['size'],
150 |                                 f['last_modified']
151 |                             ))
152 |                 else:
153 |                     # For en-US it's different, it applies to every
154 |                     # localized archives.
155 |                     # Check if they are here by listing the parent folder
156 |                     # (including en-US archive).
157 |                     l10n_parent_url = re.sub('en-US/.+$', '', url)
158 |                     l10n_folders, _ = await fetch_listing(
159 |                         session,
160 |                         l10n_parent_url,
161 |                         retry_on_notfound=True,
162 |                     )
163 |                     for locale in l10n_folders:
164 |                         _, files = await fetch_listing(
165 |                             session,
166 |                             l10n_parent_url + locale,
167 |                             retry_on_notfound=True,
168 |                         )
169 |                         for f in files:
170 |                             rc_url = l10n_parent_url + locale + f['name']
171 |                             if utils.is_build_url(product, rc_url):
172 |                                 archives.append((
173 |                                     rc_url,
174 |                                     f['size'],
175 |                                     f['last_modified'],
176 |                                 ))
177 | 
178 |                 for rc_url, size, last_modified in archives:
179 |                     record = utils.record_from_url(rc_url)
180 |                     record['download']['size'] = size
181 |                     record['download']['date'] = last_modified
182 |                     record = utils.merge_metadata(record, metadata)
183 |                     records_to_create.append(record)
184 |                 # Theorically release should never be there yet :)
185 |                 # And repacks like EME-free/sha1 don't seem to be
186 |                 # published in RC.
187 | 
188 |             # Nightly metadata
189 |             # pub/firefox/nightly/2017/08/2017-08-08-11-40-32-mozilla-central/
190 |             # firefox-57.0a1.en-US.linux-i686.json
191 |             # -l10n/...
192 |             elif utils.is_nightly_build_metadata(product, url):
193 |                 logger.info(
194 |                     f'Processing {product} nightly metadata: {key}'
195 |                 )
196 | 
197 |                 logger.debug("Fetch new nightly metadata")
198 |                 # See comment above about the exceptional need of
199 |                 # setting retry_on_notfound here.
200 |                 metadata = await fetch_json(
201 |                     session,
202 |                     url,
203 |                     retry_on_notfound=True
204 |                 )
205 | 
206 |                 platform = metadata['moz_pkg_platform']
207 | 
208 |                 # Check if english version is here.
209 |                 parent_url = re.sub('/[^/]+$', '/', url)
210 |                 logger.debug("Fetch parent listing {}".format(parent_url))
211 |                 _, files = await fetch_listing(session, parent_url)
212 |                 for f in files:
213 |                     if ('.' + platform + '.') not in f['name']:
214 |                         # metadata are by platform.
215 |                         continue
216 |                     en_nightly_url = parent_url + f['name']
217 |                     if utils.is_build_url(product, en_nightly_url):
218 |                         record = utils.record_from_url(en_nightly_url)
219 |                         record['download']['size'] = f['size']
220 |                         record['download']['date'] = f['last_modified']
221 |                         record = utils.merge_metadata(record, metadata)
222 |                         records_to_create.append(record)
223 |                         break  # Only one file for english.
224 | 
225 |                 # Check also localized versions.
226 |                 l10n_folder_url = re.sub('-mozilla-central([^/]*)/([^/]+)$',
227 |                                          '-mozilla-central\\1-l10n/',
228 |                                          url)
229 |                 logger.debug("Fetch l10n listing {}".format(l10n_folder_url))
230 |                 try:
231 |                     _, files = await fetch_listing(
232 |                         session,
233 |                         l10n_folder_url,
234 |                         retry_on_notfound=True,
235 |                     )
236 |                 except ValueError:
237 |                     files = []  # No -l10/ folder published yet.
238 |                 for f in files:
239 |                     if (
240 |                         ('.' + platform + '.') not in f['name'] and
241 |                         product != 'mobile'
242 |                     ):
243 |                         # metadata are by platform.
244 |                         # (mobile platforms are contained by folder)
245 |                         continue
246 |                     nightly_url = l10n_folder_url + f['name']
247 |                     if utils.is_build_url(product, nightly_url):
248 |                         record = utils.record_from_url(nightly_url)
249 |                         record['download']['size'] = f['size']
250 |                         record['download']['date'] = f['last_modified']
251 |                         record = utils.merge_metadata(record, metadata)
252 |                         records_to_create.append(record)
253 | 
254 |             else:
255 |                 logger.info('Ignored {}'.format(key))
256 | 
257 |             logger.debug(
258 |                 f"{len(records_to_create)} records to create."
259 |             )
260 |             with metrics.timer('s3_event_records_to_create'):
261 |                 for record in records_to_create:
262 |                     # Check that fields values look OK.
263 |                     utils.check_record(record)
264 |                     # Push result to Kinto.
265 |                     kinto_client.create_record(data=record,
266 |                                                bucket=bucket,
267 |                                                collection=collection,
268 |                                                if_not_exists=True)
269 |                     logger.info('Created {}'.format(record['id']))
270 |                     metrics.incr('s3_event_record_created')
271 | 
272 | 
273 | @sentry.capture_exceptions
274 | def lambda_handler(event, context):
275 |     # Log everything to stderr.
276 |     logger.addHandler(logging.StreamHandler(stream=sys.stdout))
277 |     logger.setLevel(logging.DEBUG)
278 | 
279 |     loop = asyncio.get_event_loop_policy().new_event_loop()
280 | 
281 |     try:
282 |         loop.run_until_complete(main(loop, event))
283 |     except Exception:
284 |         logger.exception('Aborted.')
285 |         raise
286 |     finally:
287 |         loop.close()
288 | 


--------------------------------------------------------------------------------
/jobs/buildhub/s3_inventory_to_kinto.py:
--------------------------------------------------------------------------------
  1 | # This Source Code Form is subject to the terms of the Mozilla Public
  2 | # License, v. 2.0. If a copy of the MPL was not distributed with this
  3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/.
  4 | 
  5 | import re
  6 | import asyncio
  7 | import datetime
  8 | import glob
  9 | import json
 10 | import logging
 11 | import os
 12 | import pkgutil
 13 | import tempfile
 14 | import time
 15 | import zlib
 16 | from concurrent.futures import ThreadPoolExecutor
 17 | 
 18 | import aiofiles
 19 | import aiobotocore
 20 | import botocore
 21 | from decouple import config, Csv
 22 | from aiohttp.client_exceptions import ClientPayloadError
 23 | import kinto_http
 24 | import raven
 25 | from raven.handlers.logging import SentryHandler
 26 | from ruamel import yaml
 27 | from kinto_wizard.async_kinto import AsyncKintoClient
 28 | from kinto_wizard.yaml2kinto import initialize_server
 29 | 
 30 | from buildhub.inventory_to_records import (
 31 |     __version__,
 32 |     NB_RETRY_REQUEST,
 33 |     csv_to_records,
 34 | )
 35 | from buildhub.to_kinto import fetch_existing, main as to_kinto_main
 36 | from buildhub.configure_markus import get_metrics
 37 | 
 38 | 
 39 | REGION_NAME = 'us-east-1'
 40 | BUCKET = 'net-mozaws-prod-delivery-inventory-us-east-1'
 41 | FOLDER = (
 42 |     'public/inventories/net-mozaws-prod-delivery-{inventory}/'
 43 |     'delivery-{inventory}/'
 44 | )
 45 | CHUNK_SIZE = 1024 * 256  # 256 KB
 46 | MAX_CSV_DOWNLOAD_AGE = 60 * 60 * 24 * 2  # two days
 47 | 
 48 | INITIALIZE_SERVER = config('INITIALIZE_SERVER', default=True, cast=bool)
 49 | 
 50 | # Minimum number of hours old an entry in the CSV files need to be
 51 | # to NOT be skipped.
 52 | MIN_AGE_LAST_MODIFIED_HOURS = config(
 53 |     'MIN_AGE_LAST_MODIFIED_HOURS', default=0, cast=int
 54 | )
 55 | 
 56 | CSV_DOWNLOAD_DIRECTORY = config(
 57 |     'CSV_DOWNLOAD_DIRECTORY',
 58 |     default=tempfile.gettempdir()
 59 | )
 60 | 
 61 | INVENTORIES = tuple(config(
 62 |     'INVENTORIES',
 63 |     default='firefox, archive',
 64 |     cast=Csv()
 65 | ))
 66 | 
 67 | LOG_LEVEL = config('LOG_LEVEL', default='INFO')
 68 | 
 69 | STORE_DAILY_MANIFEST = config('STORE_DAILY_MANIFEST', default=False, cast=bool)
 70 | 
 71 | # Optional Sentry with synchronuous client.
 72 | SENTRY_DSN = config('SENTRY_DSN', default=None)
 73 | sentry = raven.Client(
 74 |     SENTRY_DSN,
 75 |     transport=raven.transport.http.HTTPTransport,
 76 |     release=__version__,
 77 | )
 78 | 
 79 | logger = logging.getLogger()  # root logger.
 80 | metrics = get_metrics('buildhub')
 81 | 
 82 | 
 83 | async def initialize_kinto(loop, kinto_client, bucket, collection):
 84 |     """
 85 |     Initialize the remote server with the initialization.yml file.
 86 |     """
 87 |     # Leverage kinto-wizard async client.
 88 |     thread_pool = ThreadPoolExecutor()
 89 |     async_client = AsyncKintoClient(kinto_client, loop, thread_pool)
 90 | 
 91 |     initialization_manifest = pkgutil.get_data(
 92 |         'buildhub',
 93 |         'initialization.yml'
 94 |     )
 95 |     config = yaml.safe_load(initialization_manifest)
 96 | 
 97 |     # Check that we push the records at the right place.
 98 |     if bucket not in config:
 99 |         raise ValueError(
100 |             f"Bucket '{bucket}' not specified in `initialization.yml`."
101 |         )
102 |     if collection not in config[bucket]['collections']:
103 |         raise ValueError(
104 |             f"Collection '{collection}' not specified in `initialization.yml`."
105 |         )
106 | 
107 |     await initialize_server(async_client,
108 |                             config,
109 |                             bucket=bucket,
110 |                             collection=collection,
111 |                             force=False)
112 | 
113 | 
114 | # A regular expression corresponding to the date format in use in
115 | # delivery-firefox paths.
116 | DATE_RE = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}-\d{2}Z')
117 | 
118 | 
119 | def ends_with_date(prefix):
120 |     """Predicate to let us inspect prefixes such as:
121 | 
122 |     public/inventories/net-mozaws-prod-delivery-firefox/delivery-firefox/2017-07-01T03-09Z/
123 | 
124 |     while excluding those such as:
125 | 
126 |     public/inventories/net-mozaws-prod-delivery-firefox/delivery-firefox/hive/
127 |     """
128 |     parts = prefix.strip('/').split('/')
129 |     return DATE_RE.match(parts[-1])
130 | 
131 | 
132 | async def list_manifest_entries(loop, s3_client, inventory):
133 |     """Fetch the latest S3 inventory manifest, and the keys of every
134 |     *.csv.gz file it contains.
135 | 
136 |     :param loop: asyncio event loop.
137 |     :param s3_client: Initialized S3 client.
138 |     :param inventory str: Either "archive" or "firefox".
139 |     """
140 |     if STORE_DAILY_MANIFEST:
141 |         today_utc = datetime.datetime.utcnow().strftime('%Y%m%d')
142 |         manifest_content_file_path = f'.manifest-{today_utc}.json'
143 | 
144 |     if STORE_DAILY_MANIFEST and os.path.isfile(manifest_content_file_path):
145 |         logger.info(f"Using stored manifest file {manifest_content_file_path}")
146 |         with open(manifest_content_file_path) as f:
147 |             manifest_content = json.load(f)
148 |     else:
149 |         prefix = FOLDER.format(inventory=inventory)
150 |         paginator = s3_client.get_paginator('list_objects')
151 |         manifest_folders = []
152 |         async for result in paginator.paginate(
153 |             Bucket=BUCKET,
154 |             Prefix=prefix,
155 |             Delimiter='/'
156 |         ):
157 |             # Take latest inventory.
158 |             files = list(result.get('CommonPrefixes', []))
159 |             prefixes = [f['Prefix'] for f in files]
160 |             manifest_folders += [
161 |                 prefix for prefix in prefixes if ends_with_date(prefix)
162 |             ]
163 | 
164 |         # Download latest manifest.json
165 |         last_inventory = sorted(manifest_folders)[-1]
166 |         logger.info('Latest inventory is {}'.format(last_inventory))
167 |         key = last_inventory + 'manifest.json'
168 |         manifest = await s3_client.get_object(Bucket=BUCKET, Key=key)
169 |         async with manifest['Body'] as stream:
170 |             body = await stream.read()
171 |         manifest_content = json.loads(body.decode('utf-8'))
172 |         if STORE_DAILY_MANIFEST:
173 |             logger.info(
174 |                 f"Writing stored manifest file {manifest_content_file_path}"
175 |             )
176 |             with open(manifest_content_file_path, 'w') as f:
177 |                 json.dump(manifest_content, f, indent=3)
178 |     for f in manifest_content['files']:
179 |         # Here, each 'f' is a dictionary that looks something like this:
180 |         #
181 |         #  {
182 |         #     "key" : "inventories/net-mozaw...f-b1a0-5fb25bb83752.csv.gz",
183 |         #     "size" : 7945521,
184 |         #     "MD5checksum" : "7454b0d773000f790f15b867ee152049"
185 |         #  }
186 |         #
187 |         # We yield the whole thing. The key is used to download from S3.
188 |         # The MD5checksum is used to know how to store the file on
189 |         # disk for caching.
190 |         yield f
191 | 
192 | 
193 | async def download_csv(
194 |     loop,
195 |     s3_client,
196 |     files_stream,
197 |     chunk_size=CHUNK_SIZE,
198 |     download_directory=CSV_DOWNLOAD_DIRECTORY,
199 | ):
200 |     """
201 |     Download the S3 object of each key and return deflated data chunks (CSV).
202 |     :param loop: asyncio event loop.
203 |     :param s3_client: Initialized S3 client.
204 |     :param keys_stream async generator: List of object keys for
205 |     the csv.gz manifests.
206 |     """
207 | 
208 |     # Make sure the directory exists if it wasn't already created.
209 |     if not os.path.isdir(download_directory):
210 |         os.makedirs(download_directory, exist_ok=True)
211 | 
212 |     # Look for old download junk in the download directory.
213 |     too_old = MAX_CSV_DOWNLOAD_AGE
214 |     for file_path in glob.glob(os.path.join(download_directory, '*.csv.gz')):
215 |         age = time.time() - os.stat(file_path).st_mtime
216 |         if age > too_old:
217 |             logger.info(
218 |                 f'Delete old download file {file_path} '
219 |                 f'({age} seconds old)'
220 |             )
221 |             os.remove(file_path)
222 | 
223 |     async for files in files_stream:
224 |         # If it doesn't exist on disk, download to disk.
225 |         file_path = os.path.join(
226 |             download_directory,
227 |             files['MD5checksum'] + '.csv.gz'
228 |         )
229 |         # The file neither exists or has data.
230 |         if os.path.isfile(file_path) and os.stat(file_path).st_size:
231 |             logger.debug(f'{file_path} was already downloaded locally')
232 |         else:
233 |             key = 'public/' + files['key']
234 |             logger.info('Fetching inventory piece {}'.format(key))
235 |             file_csv_gz = await s3_client.get_object(Bucket=BUCKET, Key=key)
236 |             try:
237 |                 async with aiofiles.open(file_path, 'wb') as destination:
238 |                     async with file_csv_gz['Body'] as source:
239 |                         while 'there are chunks to read':
240 |                             gzip_chunk = await source.read(chunk_size)
241 |                             if not gzip_chunk:
242 |                                 break  # End of response.
243 |                             await destination.write(gzip_chunk)
244 |                 size = os.stat(file_path).st_size
245 |                 logger.info(f'Downloaded {key} to {file_path} ({size} bytes)')
246 |             except ClientPayloadError:
247 |                 if os.path.exists(file_path):
248 |                     os.remove(file_path)
249 |                 raise
250 | 
251 |         # Now we expect the file to exist locally. Let's read it.
252 |         gzip = zlib.decompressobj(zlib.MAX_WBITS | 16)
253 |         async with aiofiles.open(file_path, 'rb') as stream:
254 |             while 'there are chunks to read':
255 |                 gzip_chunk = await stream.read(chunk_size)
256 |                 if not gzip_chunk:
257 |                     break  # End of response.
258 |                 csv_chunk = gzip.decompress(gzip_chunk)
259 |                 if csv_chunk:
260 |                     # If the received doesn't have enough data to complete
261 |                     # at least one block, the decompressor returns an
262 |                     # empty string.
263 |                     # A later chunk added to the compressor will then
264 |                     # complete the block, it'll be decompressed and we
265 |                     # get data then.
266 |                     # Thanks Martijn Pieters http://bit.ly/2vbgQ3x
267 |                     yield csv_chunk
268 | 
269 | 
270 | async def main(loop, inventories=INVENTORIES):
271 |     """
272 |     Trigger to populate kinto with the last inventories.
273 |     """
274 |     server_url = config('SERVER_URL', default='http://localhost:8888/v1')
275 |     bucket = config('BUCKET', default='build-hub')
276 |     collection = config('COLLECTION', default='releases')
277 |     kinto_auth = tuple(config('AUTH', default='user:pass').split(':'))
278 | 
279 |     kinto_client = kinto_http.Client(server_url=server_url, auth=kinto_auth,
280 |                                      bucket=bucket, collection=collection,
281 |                                      retry=NB_RETRY_REQUEST)
282 | 
283 |     # Create bucket/collection and schemas.
284 |     if INITIALIZE_SERVER:
285 |         await initialize_kinto(loop, kinto_client, bucket, collection)
286 | 
287 |     min_last_modified = None
288 |     # Convert the simple env var integer to a datetime.datetime instance.
289 |     if MIN_AGE_LAST_MODIFIED_HOURS:
290 |         assert MIN_AGE_LAST_MODIFIED_HOURS > 0, MIN_AGE_LAST_MODIFIED_HOURS
291 |         min_last_modified = datetime.datetime.utcnow() - datetime.timedelta(
292 |             hours=MIN_AGE_LAST_MODIFIED_HOURS
293 |         )
294 |         # Make it timezone aware (to UTC)
295 |         min_last_modified = min_last_modified.replace(
296 |             tzinfo=datetime.timezone.utc
297 |         )
298 | 
299 |     # Fetch all existing records as a big dict from kinto
300 |     existing = fetch_existing(kinto_client)
301 | 
302 |     # Download CSVs, deduce records and push to Kinto.
303 |     session = aiobotocore.get_session(loop=loop)
304 |     boto_config = botocore.config.Config(signature_version=botocore.UNSIGNED)
305 |     async with session.create_client(
306 |         's3', region_name=REGION_NAME, config=boto_config
307 |     ) as client:
308 |         for inventory in inventories:
309 |             files_stream = list_manifest_entries(loop, client, inventory)
310 |             csv_stream = download_csv(loop, client, files_stream)
311 |             records_stream = csv_to_records(
312 |                 loop,
313 |                 csv_stream,
314 |                 skip_incomplete=True,
315 |                 min_last_modified=min_last_modified,
316 |             )
317 |             await to_kinto_main(
318 |                 loop,
319 |                 records_stream,
320 |                 kinto_client,
321 |                 existing=existing,
322 |                 skip_existing=False
323 |             )
324 | 
325 | 
326 | @metrics.timer_decorator('s3_inventory_to_kinto_run')
327 | def run():
328 |     # Log everything to stderr.
329 |     logger.addHandler(logging.StreamHandler())
330 |     if LOG_LEVEL.lower() == 'debug':
331 |         logger.setLevel(logging.DEBUG)
332 |     else:
333 |         logger.setLevel(logging.INFO)
334 | 
335 |     # Add Sentry (no-op if no configured).
336 |     handler = SentryHandler(sentry)
337 |     handler.setLevel(logging.ERROR)
338 |     logger.addHandler(handler)
339 | 
340 |     loop = asyncio.get_event_loop()
341 |     try:
342 |         loop.run_until_complete(main(loop))
343 |     except Exception:
344 |         logger.exception('Aborted.')
345 |         raise
346 |     finally:
347 |         loop.close()
348 | 


--------------------------------------------------------------------------------
/jobs/buildhub/to_kinto.py:
--------------------------------------------------------------------------------
  1 | # This Source Code Form is subject to the terms of the Mozilla Public
  2 | # License, v. 2.0. If a copy of the MPL was not distributed with this
  3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/.
  4 | 
  5 | """
  6 | Read records as JSON from stdin, and pushes them on a Kinto server
  7 | concurrently.
  8 | 
  9 | Usage:
 10 | 
 11 |     $ echo '{"data": {"title": "a"}}
 12 |     {"data": {"title": "b"}}
 13 |     {"data": {"title": "c"}}' | to-kinto --server=https://localhost:8888/v1 \
 14 |                                          --bucket=bid \
 15 |                                          --collection=cid \
 16 |                                          --auth=user:pass
 17 | 
 18 | It is meant to be combined with other commands that output records to stdout :)
 19 | 
 20 |     $ cat filename.csv | inventory-to-records | to-kinto --auth=user:pass
 21 |     $ scrape-archives | to-kinto --auth=user:pass
 22 | 
 23 | """
 24 | import asyncio
 25 | import async_timeout
 26 | import concurrent.futures
 27 | import copy
 28 | import hashlib
 29 | import json
 30 | import logging
 31 | import os
 32 | import sys
 33 | from urllib.parse import urlparse
 34 | 
 35 | from kinto_http import cli_utils
 36 | from decouple import config
 37 | 
 38 | from buildhub.utils import stream_as_generator
 39 | from buildhub.configure_markus import get_metrics
 40 | 
 41 | 
 42 | DEFAULT_SERVER = 'http://localhost:8888/v1'
 43 | DEFAULT_BUCKET = 'default'
 44 | DEFAULT_COLLECTION = 'cid'
 45 | NB_THREADS = 3
 46 | NB_RETRY_REQUEST = 3
 47 | WAIT_TIMEOUT = 5
 48 | BATCH_MAX_REQUESTS = config('BATCH_MAX_REQUESTS', default=9999, cast=int)
 49 | PREVIOUS_DUMP_FILENAME = '.records-hashes-{server}-{bucket}-{collection}.json'
 50 | CACHE_FOLDER = config('CACHE_FOLDER', default='.')
 51 | 
 52 | logger = logging.getLogger(__name__)
 53 | metrics = get_metrics('buildhub')
 54 | 
 55 | done = object()
 56 | 
 57 | 
 58 | def hash_record(record):
 59 |     """Return a hash string (based of MD5) that is 32 characters long.
 60 | 
 61 |     This function does *not mutate* the record but needs to make a copy of
 62 |     the record (and mutate that) so it's less performant.
 63 |     """
 64 |     return hash_record_mutate(copy.deepcopy(record))
 65 | 
 66 | 
 67 | def hash_record_mutate(record):
 68 |     """Return a hash string (based of MD5) that is 32 characters long.
 69 | 
 70 |     NOTE! For performance, this function *will mutate* the record object.
 71 |     Yeah, that sucks but it's more performant than having to clone a copy
 72 |     when you have to do it 1 million of these records.
 73 |     """
 74 |     record.pop('last_modified', None)
 75 |     record.pop('schema', None)
 76 |     return hashlib.md5(
 77 |        json.dumps(record, sort_keys=True).encode('utf-8')
 78 |     ).hexdigest()
 79 | 
 80 | 
 81 | @metrics.timer_decorator('to_kinto_fetch_existing')
 82 | def fetch_existing(
 83 |     client,
 84 |     cache_file=PREVIOUS_DUMP_FILENAME
 85 | ):
 86 |     """Fetch all records since last run. A JSON file on disk is used to store
 87 |     records from previous run.
 88 |     """
 89 |     cache_file = os.path.join(CACHE_FOLDER, cache_file.format(
 90 |         server=urlparse(client.session.server_url).hostname,
 91 |         bucket=client._bucket_name,
 92 |         collection=client._collection_name))
 93 | 
 94 |     records = {}
 95 |     previous_run_etag = None
 96 | 
 97 |     if os.path.exists(cache_file):
 98 |         with open(cache_file) as f:
 99 |             records = json.load(f)
100 |             highest_timestamp = max(
101 |                 [r[0] for r in records.values()]
102 |             )
103 |             previous_run_etag = '"%s"' % highest_timestamp
104 | 
105 |     # The reason we can't use client.get_records() is because it is not
106 |     # an iterator and if the Kinto database has 1M objects we'll end up
107 |     # with a big fat Python list of 1M objects that has repeatedly caused
108 |     # OOM errors in our stage and prod admin nodes.
109 |     if previous_run_etag:
110 |         # However, we can use it if there was a previous_run_etag which
111 |         # means we only need to extract a limited amount of records. Not
112 |         # the whole Kinto database.
113 |         new_records_batches = [client.get_records(
114 |             _since=previous_run_etag,
115 |             pages=float('inf')
116 |         )]
117 |     else:
118 |         def new_records_iterator():
119 |             params = {'_since': None}
120 |             endpoint = client.get_endpoint('records')
121 |             while True:
122 |                 record_resp, headers = client.session.request(
123 |                     'get',
124 |                     endpoint,
125 |                     params=params
126 |                 )
127 |                 yield record_resp['data']
128 |                 try:
129 |                     endpoint = headers['Next-Page']
130 |                     if not endpoint:
131 |                         raise KeyError('exists but empty value')
132 |                 except KeyError:
133 |                     break
134 |                 params.pop('_since', None)
135 | 
136 |         new_records_batches = new_records_iterator()
137 | 
138 |     count_new_records = 0
139 |     for new_records in new_records_batches:
140 |         for record in new_records:
141 |             count_new_records += 1
142 |             records[record['id']] = [
143 |                 record['last_modified'],
144 |                 hash_record_mutate(record)
145 |             ]
146 | 
147 |     metrics.gauge('to_kinto_fetched_new_records', count_new_records)
148 | 
149 |     # Atomic write.
150 |     if records:
151 |         tmpfilename = cache_file + '.tmp'
152 |         with open(tmpfilename, 'w') as f:
153 |             json.dump(records, f, sort_keys=True, indent=2)
154 |         os.rename(tmpfilename, cache_file)
155 | 
156 |     return records
157 | 
158 | 
159 | @metrics.timer_decorator('to_kinto_publish_records')
160 | def publish_records(client, records):
161 |     """Synchronuous function that pushes records on Kinto in batch.
162 |     """
163 |     with client.batch() as batch:
164 |         for record in records:
165 |             if 'id' in record['data']:
166 |                 metrics.incr('to_kinto_update_record')
167 |                 batch.update_record(**record)
168 |             else:
169 |                 metrics.incr('to_kinto_create_record')
170 |                 batch.create_record(**record)
171 |     results = batch.results()
172 | 
173 |     # Batch don't fail with 4XX errors. Make sure we output a comprehensive
174 |     # error here when we encounter them.
175 |     error_msgs = []
176 |     for result in results:
177 |         error_status = result.get('code')
178 |         if error_status == 412:
179 |             error_msg = ("Record '{details[existing][id]}' already exists: "
180 |                          '{details[existing]}').format_map(result)
181 |             error_msgs.append(error_msg)
182 |         elif error_status == 400:
183 |             error_msg = 'Invalid record: {}'.format(result)
184 |             error_msgs.append(error_msg)
185 |         elif error_status is not None:
186 |             error_msgs.append('Error: {}'.format(result))
187 |     if error_msgs:
188 |         raise ValueError('\n'.join(error_msgs))
189 | 
190 |     return results
191 | 
192 | 
193 | async def produce(loop, records, queue):
194 |     """Reads an asynchronous generator of records and puts them into the queue.
195 |     """
196 |     async for record in records:
197 |         if 'data' not in record and 'permission' not in record:
198 |             raise ValueError("Invalid record (missing 'data' attribute)")
199 | 
200 |         await queue.put(record)
201 | 
202 |     # Notify consumer that we are done.
203 |     await queue.put(done)
204 | 
205 | 
206 | async def consume(loop, queue, executor, client, existing):
207 |     """Store grabbed releases from the archives website in Kinto.
208 |     """
209 |     def markdone(queue, n):
210 |         """Returns a callback that will mark `n` queue items done."""
211 |         def done(future):
212 |             [queue.task_done() for _ in range(n)]
213 |             results = future.result()  # will raise exception if failed.
214 |             logger.info('Pushed {} records'.format(len(results)))
215 |             return results
216 |         return done
217 | 
218 |     def record_unchanged(record):
219 |         return (
220 |             record['id'] in existing and
221 |             existing.get(record['id']) == hash_record(record)
222 |         )
223 | 
224 |     info = client.server_info()
225 |     ideal_batch_size = min(
226 |         BATCH_MAX_REQUESTS,
227 |         info['settings']['batch_max_requests']
228 |     )
229 | 
230 |     while 'consumer is not cancelled':
231 |         # Consume records from queue, and batch operations.
232 |         # But don't wait too much if there's not enough records
233 |         # to fill a batch.
234 |         batch = []
235 |         try:
236 |             with async_timeout.timeout(WAIT_TIMEOUT):
237 |                 while len(batch) < ideal_batch_size:
238 |                     record = await queue.get()
239 |                     # Producer is done, don't wait for items to come in.
240 |                     if record is done:
241 |                         queue.task_done()
242 |                         break
243 |                     # Check if known and hasn't changed.
244 |                     if record_unchanged(record['data']):
245 |                         logger.debug(
246 |                             f"Skip unchanged record {record['id']}"
247 |                         )
248 |                         queue.task_done()
249 |                         continue
250 | 
251 |                     # Add record to current batch, and wait for more.
252 |                     batch.append(record)
253 | 
254 |         except asyncio.TimeoutError:
255 |             if batch:
256 |                 logger.debug(
257 |                     f'Stop waiting, proceed with {len(batch)} records.'
258 |                 )
259 |             else:
260 |                 logger.debug('Waiting for records in the queue.')
261 | 
262 |         # We have a batch of records, let's publish them using
263 |         # parallel workers.
264 |         # When done, mark queue items as done.
265 |         if batch:
266 |             task = loop.run_in_executor(
267 |                 executor, publish_records, client, batch
268 |             )
269 |             task.add_done_callback(markdone(queue, len(batch)))
270 | 
271 | 
272 | async def parse_json(lines):
273 |     async for line in lines:
274 |         record = json.loads(line.decode('utf-8'))
275 |         yield record
276 | 
277 | 
278 | async def main(
279 |     loop,
280 |     stdin_generator,
281 |     client,
282 |     skip_existing=True,
283 |     existing=None,
284 | ):
285 |     existing = existing or {}  # Because it can't be a mutable default argument
286 |     if skip_existing:
287 |         # Fetch the list of records to skip records that exist
288 |         # and haven't changed.
289 |         existing = fetch_existing(client)
290 | 
291 |     # Start a producer and a consumer with threaded kinto requests.
292 |     queue = asyncio.Queue()
293 |     executor = concurrent.futures.ThreadPoolExecutor(max_workers=NB_THREADS)
294 |     # Schedule the consumer
295 |     consumer_coro = consume(loop, queue, executor, client, existing)
296 |     consumer = asyncio.ensure_future(consumer_coro)
297 |     # Run the producer and wait for completion
298 |     await produce(loop, stdin_generator, queue)
299 |     # Wait until the consumer is done consuming everything.
300 |     await queue.join()
301 |     # The consumer is still awaiting for the producer, cancel it.
302 |     consumer.cancel()
303 | 
304 | 
305 | def run():
306 |     loop = asyncio.get_event_loop()
307 |     stdin_generator = stream_as_generator(loop, sys.stdin)
308 |     records_generator = parse_json(stdin_generator)
309 | 
310 |     parser = cli_utils.add_parser_options(
311 |         description='Read records from stdin as JSON and push them to Kinto',
312 |         default_server=DEFAULT_SERVER,
313 |         default_bucket=DEFAULT_BUCKET,
314 |         default_retry=NB_RETRY_REQUEST,
315 |         default_collection=DEFAULT_COLLECTION)
316 |     parser.add_argument('--skip', action='store_true',
317 |                         help='Skip records that exist and are equal.')
318 |     cli_args = parser.parse_args()
319 |     cli_utils.setup_logger(logger, cli_args)
320 | 
321 |     logger.info('Publish at {server}/buckets/{bucket}/collections/{collection}'
322 |                 .format(**cli_args.__dict__))
323 | 
324 |     client = cli_utils.create_client_from_args(cli_args)
325 | 
326 |     main_coro = main(
327 |         loop,
328 |         records_generator,
329 |         client,
330 |         skip_existing=cli_args.skip
331 |     )
332 | 
333 |     loop.run_until_complete(main_coro)
334 |     loop.close()
335 | 
336 | 
337 | if __name__ == '__main__':
338 |     run()
339 | 


--------------------------------------------------------------------------------
/jobs/requirements/default.txt:
--------------------------------------------------------------------------------
 1 | aiobotocore==0.5.2 \
 2 |     --hash=sha256:a06aea3aedb5e501f9625cdd10dd2645e6e70487837961c230530abe8860cfaf \
 3 |     --hash=sha256:5400ef95d4f73b06cd2bb648a33f714cea682bcfea639b2af221a4bd1b8af962
 4 | aiohttp==2.3.0 \
 5 |     --hash=sha256:4ef8aa726fec5d8fa810e61c6c42b51276c7ae962391bcdc6ac1066b49c90e7c \
 6 |     --hash=sha256:4b1d216a1ef7b7f2b06172243b1361362b94fbdbc790479c7c4f97a3c7d2e76e \
 7 |     --hash=sha256:842abbbfefbe8b9c2433c6305a533a1c361541136937e4fb3bbe70da010b5326 \
 8 |     --hash=sha256:9039c784bea791de382719056383f61e672318273cf13a8824721eb50012ebc6 \
 9 |     --hash=sha256:1ed6ce22fee3ad6d56ae139a5af0dbb6361b52ce035a4dc585a8afa5f85b394e \
10 |     --hash=sha256:a230016d972cb45dcf2b20512e39dc1f4d043eedfc181c811199ff1ef7c4df59 \
11 |     --hash=sha256:1f73f38145a6952c4b9931f759e3ff14aaddd8f1c6ac2140fd4ecec8d1b159f6 \
12 |     --hash=sha256:3441fdc7378192a482a263d3b8ba0ddcb5210076e1fb0b6005f3ae765db85ee4 \
13 |     --hash=sha256:a4d106cb3cb1fa25d822510d5cd6b3610f6d75b8d276c1c1cdd44d425efb2d63 \
14 |     --hash=sha256:371d70d62bca62da3de107a9c9f7222960755d865edd6631e530ba702ee5aa86 \
15 |     --hash=sha256:759d6dc3e8b05ac1921301ccb0557552ad7b0530e27376a6e997bc99778379ec \
16 |     --hash=sha256:e15f55a285ad46a9fbcde3d7d555691e6bebcf4e74e856cbcefb6be38f862221 \
17 |     --hash=sha256:944f9f94a9d66f2506a3f22bf9447c5b77b9ae389eead007db1d618acd157c99
18 | backoff==1.6.0 \
19 |     --hash=sha256:e3df718a774c456a516f7c88516e47a9f2d02aa562943cdfa274c439e9dbbfde
20 | kinto-http==9.1.2 \
21 |     --hash=sha256:e223e965f96e92cf916f473b4112fc4356698a553885f6dd49e99da8df00c404 \
22 |     --hash=sha256:dce1d39bad5b7323b43a2e8c8116b2dd02706e4dfa9d1554d4e573cdb9e08f01
23 | kinto-wizard==2.3.0 \
24 |     --hash=sha256:7b9a4f856b50e53079cd728feb87787b3001b0cd3b3c0ff9981a1f196adf6f2d \
25 |     --hash=sha256:c83102fcff999b4914a2bd146114566634f34a99b963885054a123b0426612a9
26 | raven==6.8.0 \
27 |     --hash=sha256:1c641e5ebc2d4185560608e253970ca0d4b98475f4edf67735015a415f9e1d48 \
28 |     --hash=sha256:95aecf76c414facaddbb056f3e98c7936318123e467728f2e50b3a66b65a6ef7
29 | markus==1.2.0 \
30 |     --hash=sha256:86bbeb16de1b1920d291c81a39b7a7c61c94b665cd8d10c6b69c994ce4fd5bcc \
31 |     --hash=sha256:9bce7bd152578703a8e4aa5a765c7c0d94bcdd69f7bc5e42d29b893e3abf2e5a
32 | datadog==0.22.0 \
33 |     --hash=sha256:86cef95acd73543d18c417f1b0313c0a7274ed8f5ae9cceb46314f4e588085b1
34 | python-decouple==3.1 \
35 |     --hash=sha256:1317df14b43efee4337a4aa02914bf004f010cd56d6c4bd894e6474ec8c4fe2d
36 | aiofiles==0.3.2 \
37 |     --hash=sha256:25c66ea3872d05d53292a6b3f7fa0f86691512076446d83a505d227b5e76f668 \
38 |     --hash=sha256:852a493a877b73e11823bfd4e8e5ef2610d70d12c9eaed961bcd9124d8de8c10
39 | ciso8601==1.0.7 \
40 |     --hash=sha256:417887f3ffd4918d758ee1e49de25c55eeb702b0f5983f84c2916aadc3071ab3
41 | pytz==2018.4 \
42 |     --hash=sha256:65ae0c8101309c45772196b21b74c46b2e5d11b6275c45d251b150d5da334555 \
43 |     --hash=sha256:c06425302f2cf668f1bba7a0a03f3c1d34d4ebeef2c72003da308b3947c7f749
44 | 


--------------------------------------------------------------------------------
/jobs/requirements/dev.txt:
--------------------------------------------------------------------------------
 1 | aioresponses==0.4.1 \
 2 |     --hash=sha256:2de01d289bacb3da18440754ebb50690b96c77f15cb546d7f78dafae9af30b50 \
 3 |     --hash=sha256:6c468badfdf71bd9bb9a49c724a176c95cc7bff38d6b0d4fc11abae1af0b8055
 4 | asynctest==0.11.1 \
 5 |     --hash=sha256:f47eb8fd1f78a63a68709c2fd471bbde038deffd4e99b8d614b988a8610c09b2 \
 6 |     --hash=sha256:f7ef31994c5e751201bd6ce6f92f60f16ad798bfaed8e2b79a74afaa4475927b
 7 | flake8==3.5.0 \
 8 |     --hash=sha256:c7841163e2b576d435799169b78703ad6ac1bbb0f199994fc05f700b2a90ea37 \
 9 |     --hash=sha256:7253265f7abd8b313e3892944044a365e3f4ac3fcdcfb4298f55ee9ddf188ba0
10 | pytest==3.5.1 \
11 |     --hash=sha256:829230122facf05a5f81a6d4dfe6454a04978ea3746853b2b84567ecf8e5c526 \
12 |     --hash=sha256:54713b26c97538db6ff0703a12b19aeaeb60b5e599de542e7fca0ec83b9038e8
13 | pytest-cache==1.0 \
14 |     --hash=sha256:be7468edd4d3d83f1e844959fd6e3fd28e77a481440a7118d430130ea31b07a9
15 | pytest-cover==3.0.0 \
16 |     --hash=sha256:5bdb6c1cc3dd75583bb7bc2c57f5e1034a1bfcb79d27c71aceb0b16af981dbf4 \
17 |     --hash=sha256:578249955eb3b5f3991209df6e532bb770b647743b7392d3d97698dc02f39ebb
18 | pytest-sugar==0.9.1 \
19 |     --hash=sha256:ab8cc42faf121344a4e9b13f39a51257f26f410e416c52ea11078cdd00d98a2c
20 | 


--------------------------------------------------------------------------------
/jobs/setup.cfg:
--------------------------------------------------------------------------------
 1 | [aliases]
 2 | test=pytest
 3 | 
 4 | [zest.releaser]
 5 | create-wheel = yes
 6 | 
 7 | [bdist_wheel]
 8 | python_tag=py3
 9 | 
10 | [flake8]
11 | max-line-length = 100
12 | 


--------------------------------------------------------------------------------
/jobs/setup.py:
--------------------------------------------------------------------------------
 1 | # This Source Code Form is subject to the terms of the Mozilla Public
 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this
 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/.
 4 | 
 5 | import codecs
 6 | import os
 7 | from setuptools import setup, find_packages
 8 | 
 9 | here = os.path.abspath(os.path.dirname(__file__))
10 | 
11 | 
12 | def read_file(filename):
13 |     """Open a related file and return its content."""
14 |     with codecs.open(os.path.join(here, filename), encoding='utf-8') as f:
15 |         content = f.read()
16 |     return content
17 | 
18 | 
19 | README = read_file('README.rst')
20 | CHANGELOG = read_file('CHANGELOG.rst')
21 | CONTRIBUTORS = read_file('CONTRIBUTORS.rst')
22 | 
23 | ENTRY_POINTS = {
24 |     'console_scripts': [
25 |         'to-kinto = buildhub.to_kinto:run',
26 |         'inventory-to-records = buildhub.inventory_to_records:run',
27 |         'latest-inventory-to-kinto = buildhub.s3_inventory_to_kinto:run',
28 |     ],
29 | }
30 | 
31 | 
32 | setup(
33 |     name='buildhub',
34 |     version='1.4.1',
35 |     description='Buildhub Python libraries.',
36 |     long_description="{}\n\n{}\n\n{}".format(README, CHANGELOG, CONTRIBUTORS),
37 |     license='MPL 2.0',
38 |     classifiers=[
39 |         "Programming Language :: Python",
40 |         "Programming Language :: Python :: 3",
41 |         "Programming Language :: Python :: 3.5",
42 |         "Programming Language :: Python :: 3.6",
43 |         "Programming Language :: Python :: Implementation :: CPython",
44 |         "Topic :: Internet :: WWW/HTTP",
45 |         "Topic :: Internet :: WWW/HTTP :: WSGI :: Application",
46 |         "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
47 |     ],
48 |     author='Mozilla Services',
49 |     author_email='storage-team@mozilla.com',
50 |     url='https://github.com/mozilla-services/buildhub',
51 |     packages=find_packages(),
52 |     package_dir={'buildhub': 'buildhub'},
53 |     package_data={'buildhub': ['initialization.yml']},
54 |     include_package_data=True,
55 |     zip_safe=False,
56 |     # Use
57 |     #  `pip -r requirements/default.txt -c requirements/constraints.txt`
58 |     # instead.
59 |     install_requires=[],
60 |     entry_points=ENTRY_POINTS
61 | )
62 | 


--------------------------------------------------------------------------------
/jobs/tests/data/inventory-simple.csv:
--------------------------------------------------------------------------------
 1 | "net-mozaws-delivery-firefox","pub/firefox/nightly/2017/05/2017-05-15-10-02-38-mozilla-central/firefox-55.0a1.en-US.linux-x86_64.tar.bz2","50000","2017-06-02T12:20:10.2Z","f1aa742ef0973db098947bd6d875f193"
 2 | "net-mozaws-delivery-firefox","pub/firefox/nightly/2017/06/2017-06-02-03-02-04-mozilla-central-l10n/firefox-55.0a1.ach.win32.complete.mar","50000","2017-06-02T12:20:10.2Z","a7fa24fe01973db09894bd6d7875f391"
 3 | "net-mozaws-delivery-firefox","pub/firefox/releases/52.0/linux-x86_64/fr/firefox-52.0.tar.bz2","60000","2017-06-02T15:20:10.4Z","1afa742ef0973db098947bd6d8759f13"
 4 | "net-mozaws-delivery-firefox","pub/firefox/releases/1.5b2/linux-i686/en-US/firefox-1.5b2.installer.tar.gz","60000","2015-10-09T15:20:10.4Z","1afa742ef0973db098947bd6d8759f13"
 5 | "net-mozaws-delivery-firefox","pub/firefox/releases/1.5b2/linux-i686/en-US/firefox-1.5b2.tar.gz","60000","2015-10-09T15:20:10.4Z","1afa742ef0973db098947bd6d8759f13"
 6 | "net-mozaws-prod-delivery-archive","pub/thunderbird/candidates/aurora-beta-channel-switch/update.mar","1502","2017-07-03T04:08:13.000Z","2c9a96bdfc62b8d43fe2f4c587454b6f"
 7 | -"net-mozaws-prod-delivery-firefox","pub/firefox/candidates/55.0b9-candidates/build2/win64/zh-TW/firefox-55.0b9.zip","53251778","2017-07-13T23:37:21.000Z","0315517f673697841a5e28ff44da2eb9"
 8 | "net-mozaws-prod-delivery-firefox","pub/firefox/candidates/55.0b9-candidates/build2/win64/zh-TW/Firefox+Setup+55.0b9.exe","37219504","2017-07-13T23:37:23.000Z","ff391f7201575ab2ba518fce20d45596"
 9 | "net-mozaws-prod-delivery-archive","pub/devedition/candidates/55.0b1-candidates/build5/win64/pt-BR/Firefox+Setup+55.0b1.exe","53718907","2017-06-14T00:41:56.000Z","8eb5c067a880ca9c63a1bf0a6c9a631d"
10 | "net-mozaws-prod-delivery-firefox","pub/firefox/releases/51.0b11/mac-EME-free/mr/Firefox+51.0b11.dmg","85984611","2017-01-04T00:05:18.000Z","9b4e5eccde90dcfd45730f810f5efc53"
11 | "net-mozaws-prod-delivery-archive","pub/mobile/nightly/2011/01/2011-01-27-03-mozilla-central-macosx/fennec-4.0b5pre.en-US.mac.dmg","18694173","2015-10-16T23:25:18.000Z","8a67ee7a7f0ce39f190bbedb4f882b00-3"
12 | "net-mozaws-delivery-firefox","pub/firefox/nightly/2018/11/2018-11-13-10-00-51-mozilla-central/firefox-65.0a1.en-US.win32.zip","50000","2018-11-13T13:10:00.51Z","f1aa742ef0973db098947bd6d875f193"
13 | "net-mozaws-delivery-firefox","pub/firefox/nightly/2018/11/2018-11-13-10-00-51-mozilla-central/firefox-65.0a1.en-US.win32.installer.exe","50000","2018-11-13T13:10:00.51Z","f1aa742ef0973db098947bd6d875f193"
14 | "net-mozaws-delivery-firefox","pub/firefox/nightly/2018/11/2018-11-13-10-00-51-mozilla-central/firefox-65.0a1.en-US.mac.dmg","50000","2018-11-13T13:10:00.51Z","f1aa742ef0973db098947bd6d875f193"
15 | "net-mozaws-delivery-firefox","pub/firefox/nightly/2018/11/2018-11-13-10-00-51-mozilla-central/firefox-65.0a1.en-US.linux-x86_64.tar.bz2","50000","2018-11-13T13:10:00.51Z","f1aa742ef0973db098947bd6d875f193"
16 | 


--------------------------------------------------------------------------------
/jobs/tests/data/s3-event-simple.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "Records": [{
 3 |         "eventTime": "2017-08-08T17:06:52.030Z",
 4 |         "s3": {
 5 |             "bucket": {"name": "archive-firefox"},
 6 |             "object": {
 7 |                 "key": "pub/firefox/releases/54.0/win64/fr/Firefox Setup 54.0.exe",
 8 |                 "size": 51001024
 9 |             }
10 |         }
11 |     }, {
12 |         "eventTime": "2017-10-29T17:06:52.030Z",
13 |         "s3": {
14 |             "bucket": {"name": "archive-firefox"},
15 |             "object": {
16 |                 "key": "pub/firefox/nightly/2017/10/2017-10-29-22-01-12-mozilla-central/firefox-58.0a1.en-US.linux-i686.tar.bz2",
17 |                 "size": 51001024
18 |             }
19 |         }
20 |     }]
21 | }


--------------------------------------------------------------------------------
/jobs/tests/test_lambda_s3_event_functional.py:
--------------------------------------------------------------------------------
  1 | # This Source Code Form is subject to the terms of the Mozilla Public
  2 | # License, v. 2.0. If a copy of the MPL was not distributed with this
  3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/.
  4 | 
  5 | import unittest
  6 | import os
  7 | import json
  8 | 
  9 | import kinto_http
 10 | from decouple import config
 11 | 
 12 | from buildhub import lambda_s3_event
 13 | 
 14 | 
 15 | here = os.path.dirname(__file__)
 16 | 
 17 | server = config('SERVER_URL', default='http://localhost:8888/v1')
 18 | bid = 'build-hub'
 19 | cid = 'releases'
 20 | 
 21 | 
 22 | class LambdaTest(unittest.TestCase):
 23 |     def setUp(self):
 24 |         filename = os.path.join(here, 'data', 's3-event-simple.json')
 25 |         self.event = json.load(open(filename, 'r'))
 26 | 
 27 |     def test_load_into_kinto(self):
 28 |         lambda_s3_event.lambda_handler(self.event, None)
 29 | 
 30 |         rid = 'firefox_54-0_win64_fr'
 31 | 
 32 |         client = kinto_http.Client(server_url=server)
 33 |         record = client.get_record(bucket=bid, collection=cid, id=rid)['data']
 34 |         record.pop('last_modified')
 35 |         assert record == {
 36 |             'id': 'firefox_54-0_win64_fr',
 37 |             'source': {
 38 |                 'repository': (
 39 |                     'https://hg.mozilla.org/releases/mozilla-release'
 40 |                 ),
 41 |                 'revision': 'e832ed037a3c23004be73178e546d240e57b6ee1',
 42 |                 'product': 'firefox',
 43 |                 'tree': 'releases/mozilla-release'
 44 |             },
 45 |             'download': {
 46 |                 'mimetype': 'application/msdos-windows',
 47 |                 'url': 'https://archive.mozilla.org/pub/firefox/releases/'
 48 |                        '54.0/win64/fr/Firefox Setup 54.0.exe',
 49 |                 'size': 51001024,
 50 |                 'date': '2017-08-08T17:06:52Z'
 51 |             },
 52 |             'target': {
 53 |                 'locale': 'fr',
 54 |                 'platform': 'win64',
 55 |                 'os': 'win',
 56 |                 'version': '54.0',
 57 |                 'channel': 'release'
 58 |             },
 59 |             'build': {
 60 |                 'as': 'ml64.exe',
 61 |                 'cc': (
 62 |                     'c:/builds/moz2_slave/m-rel-w64-00000000000000000000/'
 63 |                     'build/src/vs2015u3/VC/bin/amd64/cl.exe'
 64 |                 ),
 65 |                 'cxx': (
 66 |                     'c:/builds/moz2_slave/m-rel-w64-00000000000000000000/'
 67 |                     'build/src/vs2015u3/VC/bin/amd64/cl.exe'
 68 |                 ),
 69 |                 'date': '2017-06-08T10:58:25Z',
 70 |                 'host': 'x86_64-pc-mingw32',
 71 |                 'id': '20170608105825',
 72 |                 'number': 3,
 73 |                 'target': 'x86_64-pc-mingw32'
 74 |             }
 75 |         }
 76 | 
 77 |         rid = 'firefox_nightly_2017-10-29-22-01-12_58-0a1_linux-i686_en-us'
 78 |         record = client.get_record(bucket=bid, collection=cid, id=rid)['data']
 79 |         record.pop('last_modified')
 80 |         assert record == {
 81 |             'build': {
 82 |                 'as': '$(CC)',
 83 |                 'cc': (
 84 |                     '/usr/bin/ccache '
 85 |                     '/builds/worker/workspace/build/src/gcc/bin/gcc -m32 '
 86 |                     '-march=pentium-m -std=gnu99'
 87 |                 ),
 88 |                 'cxx': (
 89 |                     '/usr/bin/ccache '
 90 |                     '/builds/worker/workspace/build/src/gcc/bin/g++ -m32 '
 91 |                     '-march=pentium-m -std=gnu++11'
 92 |                 ),
 93 |                 'date': '2017-10-29T22:01:12Z',
 94 |                 'host': 'i686-pc-linux-gnu',
 95 |                 'id': '20171029220112',
 96 |                 'target': 'i686-pc-linux-gnu',
 97 |             },
 98 |             'download': {
 99 |                 'date': '2017-10-29T17:06:52Z',
100 |                 'mimetype': 'application/x-bzip2',
101 |                 'size': 51001024,
102 |                 'url': (
103 |                     'https://archive.mozilla.org/pub/firefox/nightly/2017/10/'
104 |                     '2017-10-29-22-01-12-mozilla-central/firefox-58.0a1.'
105 |                     'en-US.linux-i686.tar.bz2'
106 |                 )
107 |             },
108 |             'id': (
109 |                 'firefox_nightly_2017-10-29-22-01-12_58-0a1_linux-i686_en-us'
110 |             ),
111 |             'source': {
112 |                 'product': 'firefox',
113 |                 'repository': 'https://hg.mozilla.org/mozilla-central',
114 |                 'revision': 'd3910b7628b8066d3f30d58b17b5824b05768854',
115 |                 'tree': 'mozilla-central'
116 |             },
117 |             'target': {
118 |                 'channel': 'nightly',
119 |                 'locale': 'en-US',
120 |                 'os': 'linux',
121 |                 'platform': 'linux-i686',
122 |                 'version': '58.0a1'
123 |             }
124 |         }
125 | 


--------------------------------------------------------------------------------
/jobs/tests/test_s3_inventory_to_kinto.py:
--------------------------------------------------------------------------------
  1 | # This Source Code Form is subject to the terms of the Mozilla Public
  2 | # License, v. 2.0. If a copy of the MPL was not distributed with this
  3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/.
  4 | 
  5 | import base64
  6 | import json
  7 | 
  8 | import asynctest
  9 | 
 10 | from buildhub.s3_inventory_to_kinto import list_manifest_entries, download_csv
 11 | 
 12 | 
 13 | class ListManifest(asynctest.TestCase):
 14 |     def setUp(self):
 15 |         class FakePaginator:
 16 |             async def paginate(self, *args, **kwargs):
 17 |                 yield {'CommonPrefixes': [
 18 |                     {'Prefix': 'some-prefix/2017-01-02T03-05Z/'}
 19 |                 ]}
 20 |                 yield {'CommonPrefixes': [
 21 |                     {'Prefix': 'some-prefix/2017-01-02T03-04Z/'}
 22 |                 ]}
 23 |                 yield {'CommonPrefixes': [
 24 |                     {'Prefix': 'some-prefix/2017-01-02T03-06Z/'},
 25 |                     {'Prefix': 'some-prefix/data/'}
 26 |                 ]}
 27 |                 yield {'CommonPrefixes': [{'Prefix': 'some-prefix/hive/'}]}
 28 | 
 29 |         class FakeStream:
 30 |             async def __aenter__(self):
 31 |                 return self
 32 | 
 33 |             async def __aexit__(self, *args):
 34 |                 return self
 35 | 
 36 |             async def read(self):
 37 |                 return json.dumps({
 38 |                     'files': [
 39 |                         {'key': 'a/b', 'size': 1234, 'MD5checksum': 'eaf123'},
 40 |                         {'key': 'c/d', 'size': 2222, 'MD5checksum': 'ef001'},
 41 |                     ]
 42 |                 }).encode('utf-8')
 43 | 
 44 |         class FakeClient:
 45 |             def get_paginator(self, *args):
 46 |                 return FakePaginator()
 47 | 
 48 |             async def get_object(self, Bucket, Key):
 49 |                 if Key.endswith('some-prefix/2017-01-02T03-06Z/manifest.json'):
 50 |                     return {'Body': FakeStream()}
 51 | 
 52 |         self.client = FakeClient()
 53 | 
 54 |     async def test_return_keys_of_latest_manifest(self):
 55 |         results = []
 56 |         async for r in list_manifest_entries(
 57 |             self.loop,
 58 |             self.client,
 59 |             'firefox'
 60 |         ):
 61 |             results.append(r)
 62 |         assert results == [
 63 |             {'key': 'a/b', 'size': 1234, 'MD5checksum': 'eaf123'},
 64 |             {'key': 'c/d', 'size': 2222, 'MD5checksum': 'ef001'},
 65 |         ]
 66 | 
 67 | 
 68 | class DownloadCSV(asynctest.TestCase):
 69 |     def setUp(self):
 70 |         class FakeStream:
 71 |             async def __aenter__(self):
 72 |                 self.content = [
 73 |                     # echo -n "1;2;3;4\n5;6" | gzip -cf | base64
 74 |                     base64.b64decode(
 75 |                         'H4sIADPbllkAAzO0NrI2tjbhMrU2AwDZEJLXCwAAAA=='
 76 |                     ),
 77 |                     None,
 78 |                 ]
 79 |                 return self
 80 | 
 81 |             async def __aexit__(self, *args):
 82 |                 return self
 83 | 
 84 |             async def read(self, size):
 85 |                 return self.content.pop(0)
 86 | 
 87 |         class FakeClient:
 88 |             async def get_object(self, Bucket, Key):
 89 |                 if Key.endswith('public/key-1'):
 90 |                     return {'Body': FakeStream()}
 91 | 
 92 |         self.client = FakeClient()
 93 | 
 94 |     async def test_unzip_chunks(self):
 95 | 
 96 |         async def files_iterator():
 97 |             yield {
 98 |                 'key': 'key-1',
 99 |                 'size': 123456,
100 |                 'MD5checksum': 'deadbeef0123',
101 |             }
102 | 
103 |         files = files_iterator()
104 |         results = []
105 |         async for r in download_csv(self.loop, self.client, files):
106 |             results.append(r)
107 |         assert results == [b"1;2;3;4\n5;6"]
108 | 


--------------------------------------------------------------------------------
/jobs/tests/test_to_kinto.py:
--------------------------------------------------------------------------------
 1 | # This Source Code Form is subject to the terms of the Mozilla Public
 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this
 3 | # file, you can obtain one at http://mozilla.org/MPL/2.0/.
 4 | 
 5 | import unittest
 6 | # Because you can't just import unittest and access 'unittest.mock.MagicMock'
 7 | from unittest.mock import MagicMock
 8 | 
 9 | import pytest
10 | 
11 | from buildhub.to_kinto import fetch_existing
12 | 
13 | 
14 | class CacheValueTest(unittest.TestCase):
15 | 
16 |     @pytest.fixture(autouse=True)
17 |     def init_cache_files(self, tmpdir):
18 |         # Use str() on these LocalPath instances to turn them into plain
19 |         # strings since to_kinto.fetch_existing() expects it to be a string.
20 |         self.cache_file = str(tmpdir.join('cache.json'))
21 | 
22 |     def test_records_are_not_duplicated(self):
23 |         mocked = MagicMock()
24 | 
25 |         mocked.session.server_url = 'http://localhost:8888/v1'
26 |         # First, populate the cache.
27 |         mocked.session.request.return_value = (
28 |             {
29 |                 'data': [{'id': 'a', 'title': 'a', 'last_modified': 1}]
30 |             },
31 |             {}  # headers
32 |         )
33 |         first = fetch_existing(mocked, cache_file=self.cache_file)
34 |         assert isinstance(first, dict)
35 |         assert len(first) == 1
36 |         assert first['a'][0] == 1  # [0] is the last_modified
37 |         first_hash = first['a'][1]  # [1] is the hash string
38 | 
39 |         # Now that the cache file exists, it will use the regular
40 |         # client.get_records call.
41 |         mocked.get_records.return_value = [
42 |             {'id': 'a', 'title': 'b', 'last_modified': 2}
43 |         ]
44 |         second = fetch_existing(mocked, cache_file=self.cache_file)
45 |         assert isinstance(first, dict)
46 |         assert len(second) == 1
47 |         assert second['a'][0] == 2
48 |         second_hash = second['a'][1]
49 |         assert first_hash != second_hash
50 | 


--------------------------------------------------------------------------------
/kinto/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM python:3.6-slim
 2 | MAINTAINER Product Delivery irc://irc.mozilla.org/#storage-team
 3 | 
 4 | ENV PYTHONUNBUFFERED=1 \
 5 |     PYTHONPATH=/app/ \
 6 |     PORT=8888
 7 | 
 8 | EXPOSE $PORT
 9 | 
10 | # install a few essentials and clean apt caches afterwards
11 | RUN apt-get update && \
12 |     apt-get install -y --no-install-recommends \
13 |         apt-transport-https \
14 |         build-essential \
15 |         libpq-dev \
16 |         curl
17 | 
18 | # Clean up apt
19 | RUN apt-get autoremove -y && \
20 |     apt-get clean && \
21 |     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
22 | 
23 | COPY kinto/requirements.txt /tmp/
24 | WORKDIR /tmp
25 | RUN pip install --no-cache-dir -r requirements.txt
26 | 
27 | COPY . /app
28 | 
29 | # Switch back to home directory
30 | WORKDIR /app
31 | 
32 | 
33 | # Using /bin/bash as the entrypoint works around some volume mount issues on Windows
34 | # where volume-mounted files do not have execute bits set.
35 | # https://github.com/docker/compose/issues/2301#issuecomment-154450785 has additional background.
36 | ENTRYPOINT ["/bin/bash", "/app/kinto/run.sh"]
37 | 
38 | CMD ["start"]
39 | 


--------------------------------------------------------------------------------
/kinto/README.md:
--------------------------------------------------------------------------------
1 | This kinto server is for local development where you want to have a
2 | lasting PostgreSQL database with all build data in it.
3 | 


--------------------------------------------------------------------------------
/kinto/kinto.ini:
--------------------------------------------------------------------------------
  1 | # Created at Fri, 09 Mar 2018 21:12:05 +0000
  2 | # Using Kinto version 8.2.0
  3 | # Full options list for .ini file
  4 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html
  5 | 
  6 | 
  7 | [server:main]
  8 | use = egg:waitress#main
  9 | host = 0.0.0.0
 10 | port = %(http_port)s
 11 | 
 12 | 
 13 | [app:main]
 14 | use = egg:kinto
 15 | 
 16 | # Feature settings
 17 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#feature-settings
 18 | #
 19 | # kinto.readonly = false
 20 | # kinto.batch_max_requests = 25
 21 | # kinto.paginate_by =
 22 | # Experimental JSON-schema on collection
 23 | # kinto.experimental_collection_schema_validation = false
 24 | #
 25 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#activating-the-permissions-endpoint
 26 | # kinto.experimental_permissions_endpoint = false
 27 | #
 28 | # kinto.trailing_slash_redirect_enabled = true
 29 | # kinto.heartbeat_timeout_seconds = 10
 30 | 
 31 | # Plugins
 32 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#plugins
 33 | # https://github.com/uralbash/awesome-pyramid
 34 | kinto.includes = kinto.plugins.default_bucket
 35 |                  kinto_elasticsearch
 36 | #                kinto.plugins.admin
 37 | #                kinto.plugins.accounts
 38 | #                kinto.plugins.history
 39 | #                kinto.plugins.quotas
 40 | 
 41 | kinto.elasticsearch.hosts = elasticsearch:9200
 42 | # Backends
 43 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#storage
 44 | #
 45 | kinto.storage_backend = kinto.core.storage.postgresql
 46 | kinto.storage_url = postgres://postgres@db/postgres
 47 | # kinto.storage_max_fetch_size = 10000
 48 | # kinto.storage_pool_size = 25
 49 | # kinto.storage_max_overflow = 5
 50 | # kinto.storage_pool_recycle = -1
 51 | # kinto.storage_pool_timeout = 30
 52 | # kinto.storage_max_backlog = -1
 53 | 
 54 | # Cache
 55 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#cache
 56 | #
 57 | kinto.cache_backend = kinto.core.cache.postgresql
 58 | kinto.cache_url = postgres://postgres@db/postgres
 59 | # kinto.cache_prefix =
 60 | # kinto.cache_max_size_bytes = 524288
 61 | # kinto.cache_pool_size = 25
 62 | # kinto.cache_max_overflow = 5
 63 | # kinto.cache_pool_recycle = -1
 64 | # kinto.cache_pool_timeout = 30
 65 | # kinto.cache_max_backlog = -1
 66 | 
 67 | # kinto.cache_backend = kinto.core.cache.memcached
 68 | # kinto.cache_hosts = 127.0.0.1:11211
 69 | 
 70 | # Permissions.
 71 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#permissions
 72 | #
 73 | kinto.permission_backend = kinto.core.permission.postgresql
 74 | kinto.permission_url = postgres://postgres@db/postgres
 75 | # kinto.permission_pool_size = 25
 76 | # kinto.permission_max_overflow = 5
 77 | # kinto.permission_pool_recycle = 1
 78 | # kinto.permission_pool_timeout = 30
 79 | # kinto.permission_max_backlog - 1
 80 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#bypass-permissions-with-configuration
 81 | # kinto.bucket_create_principals = system.Authenticated
 82 | 
 83 | 
 84 | 
 85 | # Authentication
 86 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#authentication
 87 | #
 88 | kinto.userid_hmac_secret = 559ff64c7ad5127772bbe658655bab91fb7d82ab48ec2ccd4f0c5224caaef079
 89 | multiauth.policies = basicauth
 90 | # Any pyramid multiauth setting can be specified for custom authentication
 91 | # https://github.com/uralbash/awesome-pyramid#authentication
 92 | #
 93 | # Accounts API configuration
 94 | #
 95 | # Enable built-in plugin.
 96 | # Set `kinto.includes` to `kinto.plugins.accounts`
 97 | # Enable authenticated policy.
 98 | # Set `multiauth.policies` to `account`
 99 | # multiauth.policy.account.use = kinto.plugins.accounts.authentication.AccountsAuthenticationPolicy
100 | # Allow anyone to create accounts.
101 | # kinto.account_create_principals = system.Everyone
102 | # Set user 'account:admin' as the administrator.
103 | # kinto.account_write_principals = account:admin
104 | # kinto.account_read_principals = account:admin
105 | #
106 | # Kinto-portier authentication
107 | # https://github.com/Kinto/kinto-portier
108 | # Set `multiauth.policies` to `portier`
109 | # multiauth.policy.portier.use = kinto_portier.authentication.PortierOAuthAuthenticationPolicy
110 | # kinto.portier.broker_url = https://broker.portier.io
111 | # kinto.portier.webapp.authorized_domains = *.github.io
112 | # kinto.portier.cache_ttl_seconds = 300
113 | # kinto.portier.state.ttl_seconds = 3600
114 | 
115 | # Notifications
116 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#notifications
117 | #
118 | # Configuration example:
119 | # kinto.event_listeners = redis
120 | # kinto.event_listeners.redis.use = kinto_redis.listeners
121 | # kinto.event_listeners.redis.url = redis://localhost:6379/0
122 | # kinto.event_listeners.redis.pool_size = 5
123 | # kinto.event_listeners.redis.listname = queue
124 | # kinto.event_listeners.redis.actions = create
125 | # kinto.event_listeners.redis.resources = bucket collection
126 | 
127 | # Production settings
128 | #
129 | # https://kinto.readthedocs.io/en/latest/configuration/production.html
130 | 
131 | # kinto.http_scheme = https
132 | # kinto.http_host = kinto.services.mozilla.com
133 | 
134 | # Cross Origin Requests
135 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#cross-origin-requests-cors
136 | #
137 | # kinto.cors_origins = *
138 | 
139 | # Backoff indicators/end of service
140 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#backoff-indicators
141 | # https://kinto.readthedocs.io/en/latest/api/1.x/backoff.html#id1
142 | #
143 | # kinto.backoff =
144 | # kinto.backoff_percentage =
145 | # kinto.retry_after_seconds = 3
146 | # kinto.eos =
147 | # kinto.eos_message =
148 | # kinto.eos_url =
149 | 
150 | # Project information
151 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#project-information
152 | #
153 | # kinto.version_json_path = ./version.json
154 | # kinto.error_info_link = https://github.com/kinto/kinto/issues/
155 | # kinto.project_docs = https://kinto.readthedocs.io
156 | # kinto.project_version =
157 | # kinto.version_prefix_redirect_enabled = true
158 | 
159 | # Application profilling
160 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#application-profiling
161 | # kinto.profiler_enabled = true
162 | # kinto.profiler_dir = /tmp/profiling
163 | 
164 | # Client cache headers
165 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#client-caching
166 | #
167 | # Every bucket objects objects and list
168 | # kinto.bucket_cache_expires_seconds = 3600
169 | #
170 | # Every collection objects and list of every buckets
171 | # kinto.collection_cache_expires_seconds = 3600
172 | #
173 | # Every group objects and list of every buckets
174 | # kinto.group_cache_expires_seconds = 3600
175 | #
176 | # Every records objects and list of every collections
177 | # kinto.record_cache_expires_seconds = 3600
178 | #
179 | # Records in a specific bucket
180 | # kinto.blog_record_cache_expires_seconds = 3600
181 | #
182 | # Records in a specific collection in a specific bucket
183 | # kinto.blog_article_record_cache_expires_seconds = 3600
184 | 
185 | # Custom ID generator for POST Requests
186 | # https://kinto.readthedocs.io/en/latest/tutorials/custom-id-generator.html#tutorial-id-generator
187 | #
188 | # Default generator
189 | # kinto.bucket_id_generator=kinto.views.NameGenerator
190 | # Custom example
191 | # kinto.collection_id_generator = name_generator.CollectionGenerator
192 | # kinto.group_id_generator = name_generator.GroupGenerator
193 | # kinto.record_id_generator = name_generator.RecordGenerator
194 | 
195 | # Enabling or disabling endpoints
196 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#enabling-or-disabling-endpoints
197 | #
198 | # This is a rather confusing setting due to naming conventions used in kinto.core
199 | # For a more in depth explanation, refer to https://github.com/Kinto/kinto/issues/710
200 | # kinto.endpoint_type_resource_name_method_enabled = false
201 | # Where:
202 | # endpoint_type: is either ``collection`` (plural, e.g. ``/buckets``) or ``record`` (single, e.g. ``/buckets/abc``);
203 | # resource_name: is the name of the resource (e.g. ``bucket``, ``group``, ``collection``, ``record``);
204 | # method: is the http method (in lower case) (e.g. ``get``, ``post``, ``put``, ``patch``, ``delete``).
205 | # For example, to disable the POST on the list of buckets and DELETE on single records
206 | # kinto.collection_bucket_post_enabled = false
207 | # kinto.record_record_delete_enabled = false
208 | 
209 | # [uwsgi]
210 | # wsgi-file = app.wsgi
211 | # enable-threads = true
212 | # socket = /var/run/uwsgi/kinto.sock
213 | # chmod-socket = 666
214 | # processes =  3
215 | # master = true
216 | # module = kinto
217 | # harakiri = 120
218 | # uid = kinto
219 | # gid = kinto
220 | # virtualenv = .venv
221 | # lazy = true
222 | # lazy-apps = true
223 | # single-interpreter = true
224 | # buffer-size = 65535
225 | # post-buffering = 65535
226 | # plugin = python
227 | 
228 | # Logging and Monitoring
229 | #
230 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#logging-and-monitoring
231 | # kinto.statsd_backend = kinto.core.statsd
232 | # kinto.statsd_prefix = kinto
233 | # kinto.statsd_url =
234 | 
235 | # kinto.newrelic_config =
236 | # kinto.newrelic_env = dev
237 | 
238 | # Logging configuration
239 | 
240 | [loggers]
241 | keys = root, kinto
242 | 
243 | [handlers]
244 | keys = console
245 | 
246 | [formatters]
247 | keys = color
248 | 
249 | [logger_root]
250 | level = INFO
251 | handlers = console
252 | 
253 | [logger_kinto]
254 | level = DEBUG
255 | handlers = console
256 | qualname = kinto
257 | 
258 | [handler_console]
259 | class = StreamHandler
260 | args = (sys.stderr,)
261 | level = NOTSET
262 | formatter = color
263 | 
264 | [formatter_color]
265 | class = logging_color_formatter.ColorFormatter
266 | 


--------------------------------------------------------------------------------
/kinto/requirements.txt:
--------------------------------------------------------------------------------
1 | kinto[postgresql]
2 | kinto-wizard
3 | kinto-elasticsearch
4 | 


--------------------------------------------------------------------------------
/kinto/run.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | set -eo pipefail
 3 | 
 4 | usage() {
 5 |   echo "usage: ./bin/run.sh start|migrate|initialize-kinto-wizard"
 6 |   echo ""
 7 |   echo "    start                         Start Kinto server with memory backend"
 8 |   echo "    migrate                       Create the kinto database tables"
 9 |   echo "    initialize-kinto-wizard       Initialize a Kinto server for buildhub"
10 |   echo ""
11 |   exit 1
12 | }
13 | 
14 | 
15 | echo "THIS SHOULD WAIT TILL POSTGRES IS UP AND RUNNING." # Like Tecken
16 | 
17 | 
18 | [ $# -lt 1 ] && usage
19 | 
20 | case $1 in
21 |   start)
22 |     kinto start --ini kinto/kinto.ini ${@:2}
23 |     ;;
24 |   migrate)
25 |     kinto migrate --ini kinto/kinto.ini ${@:2}
26 |     ;;
27 |   initialize-kinto-wizard)
28 |     kinto-wizard load ${@:2}
29 |     ;;
30 |   *)
31 |     exec "$@"
32 |     ;;
33 | esac
34 | 


--------------------------------------------------------------------------------
/testkinto/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM python:3.6-slim
 2 | MAINTAINER Product Delivery irc://irc.mozilla.org/#storage-team
 3 | 
 4 | ENV PYTHONUNBUFFERED=1 \
 5 |     PYTHONPATH=/app/ \
 6 |     PORT=9999
 7 | 
 8 | EXPOSE $PORT
 9 | 
10 | # install a few essentials and clean apt caches afterwards
11 | RUN apt-get update && \
12 |     apt-get install -y --no-install-recommends \
13 |         apt-transport-https build-essential curl
14 | 
15 | # Clean up apt
16 | RUN apt-get autoremove -y && \
17 |     apt-get clean && \
18 |     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
19 | 
20 | COPY testkinto/requirements.txt /tmp/
21 | WORKDIR /tmp
22 | RUN pip install --no-cache-dir -r requirements.txt
23 | 
24 | COPY . /app
25 | 
26 | # Switch back to home directory
27 | WORKDIR /app
28 | 
29 | 
30 | # Using /bin/bash as the entrypoint works around some volume mount issues on Windows
31 | # where volume-mounted files do not have execute bits set.
32 | # https://github.com/docker/compose/issues/2301#issuecomment-154450785 has additional background.
33 | ENTRYPOINT ["/bin/bash", "/app/testkinto/run.sh"]
34 | 
35 | CMD ["start"]
36 | 


--------------------------------------------------------------------------------
/testkinto/README.md:
--------------------------------------------------------------------------------
1 | This kinto service is purely for the sake of running functional tests.
2 | The kinto server this starts uses an in-memory backend so which is
3 | convenient and safe but ephemeral.
4 | 


--------------------------------------------------------------------------------
/testkinto/kinto.ini:
--------------------------------------------------------------------------------
  1 | # Created at Fri, 09 Mar 2018 15:53:27 -0500
  2 | # Using Kinto version 8.2.0
  3 | # Full options list for .ini file
  4 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html
  5 | 
  6 | 
  7 | [server:main]
  8 | use = egg:waitress#main
  9 | host = 0.0.0.0
 10 | port = %(http_port)s
 11 | 
 12 | 
 13 | [app:main]
 14 | use = egg:kinto
 15 | 
 16 | # Feature settings
 17 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#feature-settings
 18 | #
 19 | # kinto.readonly = false
 20 | # kinto.batch_max_requests = 25
 21 | # kinto.paginate_by =
 22 | # Experimental JSON-schema on collection
 23 | # kinto.experimental_collection_schema_validation = false
 24 | #
 25 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#activating-the-permissions-endpoint
 26 | # kinto.experimental_permissions_endpoint = false
 27 | #
 28 | # kinto.trailing_slash_redirect_enabled = true
 29 | # kinto.heartbeat_timeout_seconds = 10
 30 | 
 31 | # Plugins
 32 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#plugins
 33 | # https://github.com/uralbash/awesome-pyramid
 34 | kinto.includes = kinto.plugins.default_bucket
 35 | #                kinto.plugins.admin
 36 | #                kinto.plugins.accounts
 37 | #                kinto.plugins.history
 38 | #                kinto.plugins.quotas
 39 | 
 40 | # Backends
 41 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#storage
 42 | #
 43 | kinto.storage_backend = kinto.core.storage.memory
 44 | kinto.storage_url = 
 45 | # kinto.storage_max_fetch_size = 10000
 46 | # kinto.storage_pool_size = 25
 47 | # kinto.storage_max_overflow = 5
 48 | # kinto.storage_pool_recycle = -1
 49 | # kinto.storage_pool_timeout = 30
 50 | # kinto.storage_max_backlog = -1
 51 | 
 52 | # Cache
 53 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#cache
 54 | #
 55 | kinto.cache_backend = kinto.core.cache.memory
 56 | kinto.cache_url = 
 57 | # kinto.cache_prefix =
 58 | # kinto.cache_max_size_bytes = 524288
 59 | # kinto.cache_pool_size = 25
 60 | # kinto.cache_max_overflow = 5
 61 | # kinto.cache_pool_recycle = -1
 62 | # kinto.cache_pool_timeout = 30
 63 | # kinto.cache_max_backlog = -1
 64 | 
 65 | # kinto.cache_backend = kinto.core.cache.memcached
 66 | # kinto.cache_hosts = 127.0.0.1:11211
 67 | 
 68 | # Permissions.
 69 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#permissions
 70 | #
 71 | kinto.permission_backend = kinto.core.permission.memory
 72 | kinto.permission_url = 
 73 | # kinto.permission_pool_size = 25
 74 | # kinto.permission_max_overflow = 5
 75 | # kinto.permission_pool_recycle = 1
 76 | # kinto.permission_pool_timeout = 30
 77 | # kinto.permission_max_backlog - 1
 78 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#bypass-permissions-with-configuration
 79 | # kinto.bucket_create_principals = system.Authenticated
 80 | 
 81 | # Authentication
 82 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#authentication
 83 | #
 84 | kinto.userid_hmac_secret = 5b4fde516b03e5f7645efea2ad8c7fa24440c8cd94c9133d39ffeff09049ca16
 85 | multiauth.policies = basicauth
 86 | # Any pyramid multiauth setting can be specified for custom authentication
 87 | # https://github.com/uralbash/awesome-pyramid#authentication
 88 | #
 89 | # Accounts API configuration
 90 | #
 91 | # Enable built-in plugin.
 92 | # Set `kinto.includes` to `kinto.plugins.accounts`
 93 | # Enable authenticated policy.
 94 | # Set `multiauth.policies` to `account`
 95 | # multiauth.policy.account.use = kinto.plugins.accounts.authentication.AccountsAuthenticationPolicy
 96 | # Allow anyone to create accounts.
 97 | # kinto.account_create_principals = system.Everyone
 98 | # Set user 'account:admin' as the administrator.
 99 | # kinto.account_write_principals = account:admin
100 | # kinto.account_read_principals = account:admin
101 | #
102 | # Kinto-portier authentication
103 | # https://github.com/Kinto/kinto-portier
104 | # Set `multiauth.policies` to `portier`
105 | # multiauth.policy.portier.use = kinto_portier.authentication.PortierOAuthAuthenticationPolicy
106 | # kinto.portier.broker_url = https://broker.portier.io
107 | # kinto.portier.webapp.authorized_domains = *.github.io
108 | # kinto.portier.cache_ttl_seconds = 300
109 | # kinto.portier.state.ttl_seconds = 3600
110 | 
111 | # Notifications
112 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#notifications
113 | #
114 | # Configuration example:
115 | # kinto.event_listeners = redis
116 | # kinto.event_listeners.redis.use = kinto_redis.listeners
117 | # kinto.event_listeners.redis.url = redis://localhost:6379/0
118 | # kinto.event_listeners.redis.pool_size = 5
119 | # kinto.event_listeners.redis.listname = queue
120 | # kinto.event_listeners.redis.actions = create
121 | # kinto.event_listeners.redis.resources = bucket collection
122 | 
123 | # Production settings
124 | #
125 | # https://kinto.readthedocs.io/en/latest/configuration/production.html
126 | 
127 | # kinto.http_scheme = https
128 | # kinto.http_host = kinto.services.mozilla.com
129 | 
130 | # Cross Origin Requests
131 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#cross-origin-requests-cors
132 | #
133 | # kinto.cors_origins = *
134 | 
135 | # Backoff indicators/end of service
136 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#backoff-indicators
137 | # https://kinto.readthedocs.io/en/latest/api/1.x/backoff.html#id1
138 | #
139 | # kinto.backoff =
140 | # kinto.backoff_percentage =
141 | # kinto.retry_after_seconds = 3
142 | # kinto.eos =
143 | # kinto.eos_message =
144 | # kinto.eos_url =
145 | 
146 | # Project information
147 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#project-information
148 | #
149 | # kinto.version_json_path = ./version.json
150 | # kinto.error_info_link = https://github.com/kinto/kinto/issues/
151 | # kinto.project_docs = https://kinto.readthedocs.io
152 | # kinto.project_version =
153 | # kinto.version_prefix_redirect_enabled = true
154 | 
155 | # Application profilling
156 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#application-profiling
157 | # kinto.profiler_enabled = true
158 | # kinto.profiler_dir = /tmp/profiling
159 | 
160 | # Client cache headers
161 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#client-caching
162 | #
163 | # Every bucket objects objects and list
164 | # kinto.bucket_cache_expires_seconds = 3600
165 | #
166 | # Every collection objects and list of every buckets
167 | # kinto.collection_cache_expires_seconds = 3600
168 | #
169 | # Every group objects and list of every buckets
170 | # kinto.group_cache_expires_seconds = 3600
171 | #
172 | # Every records objects and list of every collections
173 | # kinto.record_cache_expires_seconds = 3600
174 | #
175 | # Records in a specific bucket
176 | # kinto.blog_record_cache_expires_seconds = 3600
177 | #
178 | # Records in a specific collection in a specific bucket
179 | # kinto.blog_article_record_cache_expires_seconds = 3600
180 | 
181 | # Custom ID generator for POST Requests
182 | # https://kinto.readthedocs.io/en/latest/tutorials/custom-id-generator.html#tutorial-id-generator
183 | #
184 | # Default generator
185 | # kinto.bucket_id_generator=kinto.views.NameGenerator
186 | # Custom example
187 | # kinto.collection_id_generator = name_generator.CollectionGenerator
188 | # kinto.group_id_generator = name_generator.GroupGenerator
189 | # kinto.record_id_generator = name_generator.RecordGenerator
190 | 
191 | # Enabling or disabling endpoints
192 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#enabling-or-disabling-endpoints
193 | #
194 | # This is a rather confusing setting due to naming conventions used in kinto.core
195 | # For a more in depth explanation, refer to https://github.com/Kinto/kinto/issues/710
196 | # kinto.endpoint_type_resource_name_method_enabled = false
197 | # Where:
198 | # endpoint_type: is either ``collection`` (plural, e.g. ``/buckets``) or ``record`` (single, e.g. ``/buckets/abc``);
199 | # resource_name: is the name of the resource (e.g. ``bucket``, ``group``, ``collection``, ``record``);
200 | # method: is the http method (in lower case) (e.g. ``get``, ``post``, ``put``, ``patch``, ``delete``).
201 | # For example, to disable the POST on the list of buckets and DELETE on single records
202 | # kinto.collection_bucket_post_enabled = false
203 | # kinto.record_record_delete_enabled = false
204 | 
205 | # [uwsgi]
206 | # wsgi-file = app.wsgi
207 | # enable-threads = true
208 | # socket = /var/run/uwsgi/kinto.sock
209 | # chmod-socket = 666
210 | # processes =  3
211 | # master = true
212 | # module = kinto
213 | # harakiri = 120
214 | # uid = kinto
215 | # gid = kinto
216 | # virtualenv = .venv
217 | # lazy = true
218 | # lazy-apps = true
219 | # single-interpreter = true
220 | # buffer-size = 65535
221 | # post-buffering = 65535
222 | # plugin = python
223 | 
224 | # Logging and Monitoring
225 | #
226 | # https://kinto.readthedocs.io/en/latest/configuration/settings.html#logging-and-monitoring
227 | # kinto.statsd_backend = kinto.core.statsd
228 | # kinto.statsd_prefix = kinto
229 | # kinto.statsd_url =
230 | 
231 | # kinto.newrelic_config =
232 | # kinto.newrelic_env = dev
233 | 
234 | # Logging configuration
235 | 
236 | [loggers]
237 | keys = root, kinto
238 | 
239 | [handlers]
240 | keys = console
241 | 
242 | [formatters]
243 | keys = color
244 | 
245 | [logger_root]
246 | level = INFO
247 | handlers = console
248 | 
249 | [logger_kinto]
250 | level = DEBUG
251 | handlers = console
252 | qualname = kinto
253 | 
254 | [handler_console]
255 | class = StreamHandler
256 | args = (sys.stderr,)
257 | level = NOTSET
258 | formatter = color
259 | 
260 | [formatter_color]
261 | class = logging_color_formatter.ColorFormatter
262 | 


--------------------------------------------------------------------------------
/testkinto/requirements.txt:
--------------------------------------------------------------------------------
1 | kinto
2 | kinto-wizard
3 | 


--------------------------------------------------------------------------------
/testkinto/run.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | set -eo pipefail
 3 | 
 4 | usage() {
 5 |   echo "usage: ./bin/run.sh start|initialize-kinto-wizard"
 6 |   echo ""
 7 |   echo "    start                         Start Kinto server with memory backend"
 8 |   echo "    initialize-kinto-wizard       Initialize a Kinto server for buildhub"
 9 |   echo ""
10 |   exit 1
11 | }
12 | 
13 | [ $# -lt 1 ] && usage
14 | 
15 | case $1 in
16 |   start)
17 |     # Note that this kinto.ini is part of the code.
18 |     kinto start --ini testkinto/kinto.ini --port 9999 ${@:2}
19 |     ;;
20 |   initialize-kinto-wizard)
21 |     kinto-wizard load ${@:2}
22 |     ;;
23 |   *)
24 |     exec "$@"
25 |     ;;
26 | esac
27 | 


--------------------------------------------------------------------------------
/ui/.gitignore:
--------------------------------------------------------------------------------
1 | /build
2 | /node_modules


--------------------------------------------------------------------------------
/ui/Dockerfile:
--------------------------------------------------------------------------------
 1 | # Note! If you make changes it in this file, to rebuild it use:
 2 | #   docker-compose build ui
 3 | #
 4 | 
 5 | FROM node:9
 6 | 
 7 | ADD ui/package.json /package.json
 8 | ADD ui/yarn.lock /yarn.lock
 9 | RUN yarn
10 | 
11 | ENV NODE_PATH=/node_modules
12 | ENV PATH=$PATH:/node_modules/.bin
13 | 
14 | WORKDIR /app
15 | ADD ui /app
16 | 
17 | EXPOSE 3000
18 | EXPOSE 35729
19 | 
20 | ENTRYPOINT ["/bin/bash", "/app/run.sh"]
21 | 
22 | CMD ["start"]
23 | 


--------------------------------------------------------------------------------
/ui/README.md:
--------------------------------------------------------------------------------
 1 | # BuildKit
 2 | 
 3 | A SearchKit-powered [Web UI](https://mozilla-services.github.io/buildhub/searchkit/) to browse Mozilla builds information.
 4 | 
 5 | ## Install
 6 | 
 7 |     $ yarn
 8 |     $ yarn start
 9 | 
10 | ## Build
11 | 
12 |     $ yarn run build
13 | 
14 | ## Deploy
15 | 
16 |     $ yarn run deploy
17 | 
18 | App should be available at https://mozilla-services.github.io/buildhub.
19 | 


--------------------------------------------------------------------------------
/ui/lint_problems.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | set -eo pipefail
 3 | 
 4 | prelude() {
 5 |   echo "
 6 | You have prettier linting errors!
 7 | ----------------------------------
 8 | The following files would turn out different if you process them with prettier:
 9 | "
10 | }
11 | 
12 | any=false
13 | first=true
14 | while read line
15 | do
16 |   $first && prelude
17 |   echo "$line"
18 |   echo ""
19 |   echo "To fix:"
20 |   echo "    prettier --write ${line}"
21 |   echo "To see:"
22 |   echo "    prettier ${line} | diff ${line} -"
23 |   any=true
24 |   first=false
25 | done < "${1:-/dev/stdin}"
26 | 
27 | 
28 | $any && echo "
29 | If you're not interested in how they're different, consider running:
30 | 
31 |   yarn run lint:prettierfix
32 | "
33 | 
34 | $any && exit 1 || exit 0
35 | 


--------------------------------------------------------------------------------
/ui/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "elastic",
 3 |   "version": "0.1.0",
 4 |   "private": true,
 5 |   "dependencies": {
 6 |     "@searchkit/refinement-autosuggest": "0.0.1-6",
 7 |     "filesize": "3.5.10",
 8 |     "react": "15.5.4",
 9 |     "react-dom": "15.5.4",
10 |     "rimraf": "2.6.1",
11 |     "searchkit": "2.3.0-9"
12 |   },
13 |   "devDependencies": {
14 |     "gh-pages": "1.0.0",
15 |     "prettier": "1.11.1",
16 |     "react-scripts": "1.0.7"
17 |   },
18 |   "resolutions": {
19 |     "querystringify": "2.0.0",
20 |     "nwmatcher": "1.4.4",
21 |     "handlebars": "4.0.14",
22 |     "js-yaml": "3.13.1",
23 |     "fstream": "1.0.12",
24 |     "axios": "0.18.1",
25 |     "tar": "2.2.2",
26 |     "eslint": "4.18.2"
27 |   },
28 |   "homepage": "https://mozilla-services.github.io/buildhub",
29 |   "scripts": {
30 |     "deploy":
31 |       "yarn run build && rimraf tmp && mkdir tmp && cp -R build/* tmp/ && gh-pages -d tmp --add && rimraf tmp",
32 |     "start": "react-scripts start",
33 |     "build": "react-scripts build",
34 |     "test": "react-scripts test --env=jsdom",
35 |     "eject": "react-scripts eject",
36 |     "lint:prettier":
37 |       "prettier --list-different src/**/*.js | ./lint_problems.sh",
38 |     "lint:prettierfix": "prettier src/**/*.js --write"
39 |   },
40 |   "proxy": "http://kinto:8888"
41 | }
42 | 


--------------------------------------------------------------------------------
/ui/public/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mozilla-services/buildhub/82919da2d31bd645948cb4e8281c0d8c83c707a3/ui/public/favicon.ico


--------------------------------------------------------------------------------
/ui/public/index.html:
--------------------------------------------------------------------------------
 1 | <!doctype html>
 2 | <html lang="en">
 3 |   <head>
 4 |     <meta charset="utf-8">
 5 |     <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
 6 |     <meta name="theme-color" content="#000000">
 7 |     <!--
 8 |       manifest.json provides metadata used when your web app is added to the
 9 |       homescreen on Android. See https://developers.google.com/web/fundamentals/engage-and-retain/web-app-manifest/
10 |     -->
11 |     <link rel="manifest" href="%PUBLIC_URL%/manifest.json">
12 |     <link rel="shortcut icon" href="%PUBLIC_URL%/favicon.ico">
13 |     <!--
14 |       Notice the use of %PUBLIC_URL% in the tags above.
15 |       It will be replaced with the URL of the `public` folder during the build.
16 |       Only files inside the `public` folder can be referenced from the HTML.
17 | 
18 |       Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
19 |       work correctly both with client-side routing and a non-root public URL.
20 |       Learn how to configure a non-root public URL by running `npm run build`.
21 |     -->
22 |     <title>Mozilla Build Metadata Service</title>
23 |   </head>
24 |   <body>
25 |     <noscript>
26 |       You need to enable JavaScript to run this app.
27 |     </noscript>
28 |     <div id="root"></div>
29 |   </body>
30 | </html>
31 | 


--------------------------------------------------------------------------------
/ui/public/manifest.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "short_name": "React App",
 3 |   "name": "Create React App Sample",
 4 |   "icons": [
 5 |     {
 6 |       "src": "favicon.ico",
 7 |       "sizes": "192x192",
 8 |       "type": "image/png"
 9 |     }
10 |   ],
11 |   "start_url": "./index.html",
12 |   "display": "standalone",
13 |   "theme_color": "#000000",
14 |   "background_color": "#ffffff"
15 | }
16 | 


--------------------------------------------------------------------------------
/ui/run.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | set -eo pipefail
 3 | 
 4 | usage() {
 5 |   echo "usage: ./run.sh start|outdated|lintcheck|lintfix"
 6 |   echo ""
 7 |   echo "    start                         Start React dev server"
 8 |   echo "    outdated                      List npm packages that are outdated"
 9 |   echo "    lintcheck                     Prettier check all the source files"
10 |   echo "    lintfix                       Let Prettier fix all source files"
11 |   echo ""
12 |   exit 1
13 | }
14 | 
15 | [ $# -lt 1 ] && usage
16 | 
17 | 
18 | case $1 in
19 |   start)
20 |     yarn run start | cat
21 |     ;;
22 |   outdated)
23 |     yarn outdated
24 |     ;;
25 |   lintcheck)
26 |     yarn run lint:prettier
27 |     ;;
28 |   lintfix)
29 |     yarn run lint:prettierfix
30 |     ;;
31 |   *)
32 |     exec "$@"
33 |     ;;
34 | esac
35 | 


--------------------------------------------------------------------------------
/ui/src/App.css:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mozilla-services/buildhub/82919da2d31bd645948cb4e8281c0d8c83c707a3/ui/src/App.css


--------------------------------------------------------------------------------
/ui/src/App.js:
--------------------------------------------------------------------------------
  1 | /* This Source Code Form is subject to the terms of the Mozilla Public
  2 |  * License, v. 2.0. If a copy of the MPL was not distributed with this
  3 |  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
  4 | 
  5 | import React, { Component } from "react";
  6 | import "./App.css";
  7 | import filesize from "filesize";
  8 | 
  9 | import {
 10 |   SearchBox,
 11 |   NoHits,
 12 |   Hits,
 13 |   HitsStats,
 14 |   SortingSelector,
 15 |   SelectedFilters,
 16 |   MenuFilter,
 17 |   Pagination,
 18 |   ResetFilters,
 19 |   SearchkitManager,
 20 |   SearchkitProvider,
 21 |   Tabs
 22 | } from "searchkit";
 23 | 
 24 | import { RefinementAutosuggest } from "@searchkit/refinement-autosuggest";
 25 | 
 26 | import {
 27 |   Layout,
 28 |   TopBar,
 29 |   LayoutBody,
 30 |   LayoutResults,
 31 |   ActionBar,
 32 |   ActionBarRow,
 33 |   SideBar
 34 | } from "searchkit";
 35 | 
 36 | import "searchkit/release/theme.css";
 37 | 
 38 | import contribute_json from "./contribute.json";
 39 | 
 40 | let defaultCollectionURL =
 41 |   "https://buildhub.prod.mozaws.net/v1/buckets/build-hub/collections/releases/";
 42 | // There is an exception to this rule, the default (when
 43 | // REACT_APP_KINTO_COLLECTION_URL isn't set) depends on how this UI is rendered.
 44 | if (
 45 |   window.location.pathname &&
 46 |   window.location.pathname.search(/\/stage\//) > -1
 47 | ) {
 48 |   defaultCollectionURL =
 49 |     "https://buildhub.stage.mozaws.net/v1/buckets/build-hub/collections/releases/";
 50 | }
 51 | const KINTO_COLLECTION_URL =
 52 |   process.env.REACT_APP_KINTO_COLLECTION_URL || defaultCollectionURL;
 53 | 
 54 | const searchkit = new SearchkitManager(KINTO_COLLECTION_URL, {
 55 |   searchUrlPath: "search"
 56 | });
 57 | 
 58 | const HitsTable = ({ hits }) => {
 59 |   return (
 60 |     <div style={{ width: "100%", boxSizing: "border-box", padding: 8 }}>
 61 |       <table
 62 |         className="sk-table sk-table-striped"
 63 |         style={{ width: "100%", boxSizing: "border-box" }}
 64 |       >
 65 |         <thead>
 66 |           <tr>
 67 |             <th />
 68 |             <th>Product</th>
 69 |             <th>Version</th>
 70 |             <th>platform</th>
 71 |             <th>channel</th>
 72 |             <th>locale</th>
 73 |             <th>Tree</th>
 74 |             <th>Size</th>
 75 |             <th>Published on</th>
 76 |             <th>Build ID</th>
 77 |             <th>Revision</th>
 78 |           </tr>
 79 |         </thead>
 80 |         <tbody>
 81 |           {hits.map(
 82 |             ({
 83 |               _source: { build, download, source, target },
 84 |               _id,
 85 |               highlight
 86 |             }) => {
 87 |               const recordUrl = `${KINTO_COLLECTION_URL}records/${_id}`;
 88 |               const revisionUrl = source.revision ? (
 89 |                 <a href={`${source.repository}/rev/${source.revision}`}>
 90 |                   {source.revision.substring(0, 6)}
 91 |                 </a>
 92 |               ) : (
 93 |                 ""
 94 |               );
 95 |               const getHighlight = (title, value) => {
 96 |                 return { __html: (highlight && highlight[title]) || value };
 97 |               };
 98 |               return (
 99 |                 <tr key={_id} id={_id}>
100 |                   <td>
101 |                     <a href={`#${_id}`}>#</a>
102 |                   </td>
103 |                   <td
104 |                     dangerouslySetInnerHTML={getHighlight(
105 |                       "source.product",
106 |                       source.product
107 |                     )}
108 |                   />
109 |                   <td
110 |                     dangerouslySetInnerHTML={getHighlight(
111 |                       "target.version",
112 |                       target.version
113 |                     )}
114 |                   />
115 |                   <td
116 |                     dangerouslySetInnerHTML={getHighlight(
117 |                       "target.platform",
118 |                       target.platform
119 |                     )}
120 |                   />
121 |                   <td
122 |                     dangerouslySetInnerHTML={getHighlight(
123 |                       "target.channel",
124 |                       target.channel
125 |                     )}
126 |                   />
127 |                   <td
128 |                     dangerouslySetInnerHTML={getHighlight(
129 |                       "target.locale",
130 |                       target.locale
131 |                     )}
132 |                   />
133 |                   <td>{source.tree}</td>
134 |                   <td>
135 |                     <a href={download.url}>{filesize(download.size)}</a>
136 |                   </td>
137 |                   <td title={download.date}>
138 |                     <a
139 |                       target="_blank"
140 |                       rel="noopener noreferrer"
141 |                       href={recordUrl}
142 |                     >
143 |                       <time dateTime={download.date}>{download.date}</time>
144 |                     </a>
145 |                   </td>
146 |                   <td
147 |                     dangerouslySetInnerHTML={getHighlight(
148 |                       "build.id",
149 |                       build && build.id
150 |                     )}
151 |                   />
152 |                   <td>{revisionUrl}</td>
153 |                 </tr>
154 |               );
155 |             }
156 |           )}
157 |         </tbody>
158 |       </table>
159 |     </div>
160 |   );
161 | };
162 | 
163 | const fullText = (query, options) => {
164 |   if (!query) {
165 |     return;
166 |   }
167 |   const fulltextQuery = query.startsWith("'")
168 |     ? query.slice(1)
169 |     : query
170 |         .split(" ")
171 |         .map(term => {
172 |           return `${term}*`;
173 |         })
174 |         .join(" ");
175 |   return {
176 |     query_string: Object.assign({ query: fulltextQuery }, options)
177 |   };
178 | };
179 | 
180 | class Deprecation extends React.PureComponent {
181 |   render() {
182 |     return (
183 |       <div className="deprecation-notice">
184 |         This project is deprecated and will be decommissioned soon.{" "}
185 |         <a href="https://github.com/mozilla-services/buildhub/issues/532">
186 |           See deprecation notice for details.
187 |         </a>
188 |       </div>
189 |     );
190 |   }
191 | }
192 | 
193 | class ProjectInfo extends React.PureComponent {
194 |   render() {
195 |     const {
196 |       repository: { url: source, license },
197 |       participate: { docs: documentation },
198 |       bugs: { report }
199 |     } = contribute_json;
200 | 
201 |     return (
202 |       <div className="project-info">
203 |         <div>
204 |           <a href={documentation}>Docs</a>
205 |         </div>
206 |         <div>
207 |           <a href={report}>Bugs</a>
208 |         </div>
209 |         <div>
210 |           <a href={source} title={license}>
211 |             Source
212 |           </a>
213 |         </div>
214 |       </div>
215 |     );
216 |   }
217 | }
218 | 
219 | class App extends Component {
220 |   render() {
221 |     return (
222 |       <div className="App">
223 |         <SearchkitProvider searchkit={searchkit}>
224 |           <Layout>
225 |             <TopBar>
226 |               <SearchBox
227 |                 autofocus={true}
228 |                 searchOnChange={true}
229 |                 placeholder="e.g. firefox 54 linux"
230 |                 queryBuilder={fullText}
231 |                 queryOptions={{
232 |                   analyzer: "standard",
233 |                   default_operator: "AND",
234 |                   phrase_slop: 1,
235 |                   auto_generate_phrase_queries: true,
236 |                   analyze_wildcard: true,
237 |                   lenient: true,
238 |                   split_on_whitespace: true
239 |                 }}
240 |                 queryFields={[
241 |                   "source.product",
242 |                   "target.channel^1.2",
243 |                   "target.version^10",
244 |                   "target.locale^3",
245 |                   "target.platform^2",
246 |                   "build.id"
247 |                 ]}
248 |               />
249 |               <div className="elasticsearch-query-doc">
250 |                 <a
251 |                   href="https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax"
252 |                   title="You may use Elasticsearch query string syntax by prefixing it with a single quote"
253 |                 >
254 |                   ?
255 |                 </a>
256 |               </div>
257 |               <ProjectInfo />
258 |             </TopBar>
259 |             <LayoutBody>
260 |               <SideBar>
261 |                 <RefinementAutosuggest
262 |                   field="target.version"
263 |                   title="Version"
264 |                   git
265 |                   id="versions"
266 |                   size={1000}
267 |                   operator="OR"
268 |                   multi={true}
269 |                   orderKey={"_term"}
270 |                   orderDirection={"desc"}
271 |                 />
272 |                 <RefinementAutosuggest
273 |                   field="target.platform"
274 |                   title="Platform"
275 |                   id="platform"
276 |                   size={1000}
277 |                   orderKey="_term"
278 |                   multi={true}
279 |                 />
280 |                 <RefinementAutosuggest
281 |                   field="target.channel"
282 |                   title="Channel"
283 |                   id="channel"
284 |                   size={1000}
285 |                   orderKey="_term"
286 |                   multi={true}
287 |                 />
288 |                 <RefinementAutosuggest
289 |                   field="target.locale"
290 |                   title="Locale"
291 |                   id="locale"
292 |                   size={1000}
293 |                   orderKey="_term"
294 |                   multi={true}
295 |                 />
296 |               </SideBar>
297 | 
298 |               <LayoutResults>
299 |                 <Deprecation />
300 |                 <ActionBar>
301 |                   <ActionBarRow>
302 |                     <HitsStats />
303 |                     <SortingSelector
304 |                       options={[
305 |                         {
306 |                           label: "Published on",
307 |                           field: "download.date",
308 |                           order: "desc",
309 |                           defaultOption: true
310 |                         },
311 |                         {
312 |                           label: "Build date",
313 |                           field: "build.date",
314 |                           order: "desc"
315 |                         }
316 |                       ]}
317 |                     />
318 |                   </ActionBarRow>
319 | 
320 |                   <ActionBarRow>
321 |                     <SelectedFilters />
322 |                     <ResetFilters />
323 |                   </ActionBarRow>
324 | 
325 |                   <MenuFilter
326 |                     field="source.product"
327 |                     title="Product"
328 |                     id="products"
329 |                     listComponent={Tabs}
330 |                     translations={{ All: "All products" }}
331 |                   />
332 |                 </ActionBar>
333 | 
334 |                 <Hits
335 |                   hitsPerPage={30}
336 |                   listComponent={HitsTable}
337 |                   highlightFields={[
338 |                     "source.product",
339 |                     "target.channel",
340 |                     "target.version",
341 |                     "target.locale",
342 |                     "target.platform",
343 |                     "build.id"
344 |                   ]}
345 |                 />
346 |                 <NoHits
347 |                   translations={{
348 |                     "NoHits.NoResultsFound":
349 |                       "No release found were found for {query}",
350 |                     "NoHits.DidYouMean": "Search for {suggestion}",
351 |                     "NoHits.SearchWithoutFilters":
352 |                       "Search for {query} without filters"
353 |                   }}
354 |                   suggestionsField="target.version"
355 |                 />
356 | 
357 |                 <Pagination showNumbers={true} />
358 |               </LayoutResults>
359 |             </LayoutBody>
360 |           </Layout>
361 |         </SearchkitProvider>
362 |       </div>
363 |     );
364 |   }
365 | }
366 | 
367 | export default App;
368 | 


--------------------------------------------------------------------------------
/ui/src/App.test.js:
--------------------------------------------------------------------------------
 1 | /* This Source Code Form is subject to the terms of the Mozilla Public
 2 |  * License, v. 2.0. If a copy of the MPL was not distributed with this
 3 |  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 4 | 
 5 | import React from "react";
 6 | import ReactDOM from "react-dom";
 7 | import App from "./App";
 8 | 
 9 | it("renders without crashing", () => {
10 |   const div = document.createElement("div");
11 |   ReactDOM.render(<App />, div);
12 | });
13 | 


--------------------------------------------------------------------------------
/ui/src/contribute.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "name": "buildhub",
 3 |     "description": "Mozilla Build Metadata Service",
 4 |     "participate": {
 5 |         "irc": "irc://irc.mozilla.org/#storage-team",
 6 |         "irc-contacts": ["leplatrem", "magopian", "mostlygeek"],
 7 |         "docs": "https://buildhub.readthedocs.io/en/latest/"
 8 |     },
 9 |     "repository": {
10 |         "url": "https://github.com/mozilla-services/buildhub",
11 |         "license": "MPL 2.0"
12 |     },
13 |     "bugs": {
14 |         "list": "https://github.com/mozilla-services/buildhub/issues",
15 |         "report": "https://github.com/mozilla-services/buildhub/issues/new"
16 |     },
17 |     "urls": {
18 |         "prod": "https://mozilla-services.github.io/buildhub/",
19 |         "stage": "https://mozilla-services.github.io/buildhub/stage/"
20 |     }
21 | }
22 | 


--------------------------------------------------------------------------------
/ui/src/index.css:
--------------------------------------------------------------------------------
 1 | body {
 2 |   margin: 0;
 3 |   padding: 0;
 4 |   font-family: sans-serif;
 5 | }
 6 | 
 7 | .elasticsearch-query-doc {
 8 |   font-size: 16px;
 9 |   margin-top: 7px;
10 | }
11 | 
12 | .elasticsearch-query-doc a {
13 |   color: #fff;
14 |   padding: 20px 15px 20px 15px;
15 | }
16 | 
17 | .project-info {
18 |   color: #fff;
19 | }
20 | .project-info > div {
21 |   display: inline-block;
22 |   margin-right: 20px;
23 |   margin-top: 7px;
24 | }
25 | .project-info a {
26 |   color: #fff;
27 | }
28 | 
29 | .deprecation-notice {
30 |   margin: 20px;
31 |   padding: 5px;
32 |   background: #ffaaaa;
33 |   font-weight: bold;
34 | }
35 | 
36 | table a {
37 |   text-decoration: none;
38 | }
39 | 
40 | table em {
41 |   background-color: yellow;
42 | }
43 | 
44 | .sk-table td, .sk-table th {
45 |   padding: .5em .5em;
46 |   text-overflow: ellipsis;
47 |   white-space: nowrap;
48 | }
49 | 


--------------------------------------------------------------------------------
/ui/src/index.js:
--------------------------------------------------------------------------------
 1 | /* This Source Code Form is subject to the terms of the Mozilla Public
 2 |  * License, v. 2.0. If a copy of the MPL was not distributed with this
 3 |  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 4 | 
 5 | import React from "react";
 6 | import ReactDOM from "react-dom";
 7 | import App from "./App";
 8 | import registerServiceWorker from "./registerServiceWorker";
 9 | import "./index.css";
10 | 
11 | ReactDOM.render(<App />, document.getElementById("root"));
12 | registerServiceWorker();
13 | 


--------------------------------------------------------------------------------
/ui/src/registerServiceWorker.js:
--------------------------------------------------------------------------------
 1 | // In production, we register a service worker to serve assets from local cache.
 2 | 
 3 | // This lets the app load faster on subsequent visits in production, and gives
 4 | // it offline capabilities. However, it also means that developers (and users)
 5 | // will only see deployed updates on the "N+1" visit to a page, since previously
 6 | // cached resources are updated in the background.
 7 | 
 8 | // To learn more about the benefits of this model, read https://goo.gl/KwvDNy.
 9 | // This link also includes instructions on opting out of this behavior.
10 | 
11 | export default function register() {
12 |   if (process.env.NODE_ENV === "production" && "serviceWorker" in navigator) {
13 |     window.addEventListener("load", () => {
14 |       const swUrl = `${process.env.PUBLIC_URL}/service-worker.js`;
15 |       navigator.serviceWorker
16 |         .register(swUrl)
17 |         .then(registration => {
18 |           registration.onupdatefound = () => {
19 |             const installingWorker = registration.installing;
20 |             installingWorker.onstatechange = () => {
21 |               if (installingWorker.state === "installed") {
22 |                 if (navigator.serviceWorker.controller) {
23 |                   // At this point, the old content will have been purged and
24 |                   // the fresh content will have been added to the cache.
25 |                   // It's the perfect time to display a "New content is
26 |                   // available; please refresh." message in your web app.
27 |                   console.log("New content is available; please refresh.");
28 |                 } else {
29 |                   // At this point, everything has been precached.
30 |                   // It's the perfect time to display a
31 |                   // "Content is cached for offline use." message.
32 |                   console.log("Content is cached for offline use.");
33 |                 }
34 |               }
35 |             };
36 |           };
37 |         })
38 |         .catch(error => {
39 |           console.error("Error during service worker registration:", error);
40 |         });
41 |     });
42 |   }
43 | }
44 | 
45 | export function unregister() {
46 |   if ("serviceWorker" in navigator) {
47 |     navigator.serviceWorker.ready.then(registration => {
48 |       registration.unregister();
49 |     });
50 |   }
51 | }
52 | 


--------------------------------------------------------------------------------