├── AUTHORS.txt ├── CONTRIBUTING.md ├── CONTRIBUTORS.txt ├── LICENSE.txt ├── README.md ├── containers ├── base │ ├── .dockerignore │ ├── .gitignore │ ├── dev-requirements.txt │ ├── javascript │ │ ├── .bowerrc │ │ ├── .dockerignore │ │ ├── .gitignore │ │ ├── .jshintrc │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── THOUGHTS.txt │ │ ├── bower.json │ │ ├── browser │ │ │ ├── css │ │ │ │ └── dashboard.css │ │ │ ├── html │ │ │ │ └── client.html │ │ │ ├── img │ │ │ │ ├── drop_file_icon.svg │ │ │ │ ├── favicon.ico │ │ │ │ ├── fb_logo.svg │ │ │ │ ├── font_icon.svg │ │ │ │ ├── logo.png │ │ │ │ ├── logo_200px.png │ │ │ │ ├── logo_icon.png │ │ │ │ ├── oven_icon.svg │ │ │ │ └── report_icon.svg │ │ │ └── js │ │ │ │ ├── CollectionController.js │ │ │ │ ├── CollectionReport.js │ │ │ │ ├── Controller.js │ │ │ │ ├── DashboardController.js │ │ │ │ ├── DispatcherController.js │ │ │ │ ├── DispatcherLists.js │ │ │ │ ├── PubSub.js │ │ │ │ ├── Report.js │ │ │ │ ├── StatusController.js │ │ │ │ ├── UserLog.js │ │ │ │ ├── bootstrap.js │ │ │ │ ├── dom-tool.js │ │ │ │ ├── exsample.json.js │ │ │ │ ├── isEqual.js │ │ │ │ ├── main.js │ │ │ │ ├── reporterBlocks.js │ │ │ │ └── xhr.js │ │ ├── generated_modules │ │ │ └── protocolbuffers │ │ │ │ ├── fonts_public_grpc_pb.js │ │ │ │ ├── fonts_public_pb.js │ │ │ │ ├── messages_grpc_pb.js │ │ │ │ ├── messages_pb.js │ │ │ │ ├── shared_grpc_pb.js │ │ │ │ └── shared_pb.js │ │ ├── node │ │ │ ├── CacheServer.js │ │ │ ├── GitHubAuthServer.js │ │ │ ├── GitHubOperationsServer.js │ │ │ ├── InitWorkers.js │ │ │ ├── ManifestMaster.js │ │ │ ├── PersistenceServer.js │ │ │ ├── ReportsServer.js │ │ │ ├── _BaseServer.js │ │ │ ├── api.js │ │ │ ├── apiServices │ │ │ │ ├── GithubOAuth.js │ │ │ │ ├── StorageBrowse.js │ │ │ │ └── StorageDownload.js │ │ │ ├── collectiontests.js │ │ │ ├── dispatcher │ │ │ │ ├── DispatcherProcessManager.js │ │ │ │ ├── DummyDispatcherProcessManager.js │ │ │ │ ├── FamilyPRDispatcherProcess.js │ │ │ │ ├── ProcessUIService.js │ │ │ │ ├── concept.md │ │ │ │ └── framework │ │ │ │ │ ├── GenericProcess.js │ │ │ │ │ ├── Path.js │ │ │ │ │ ├── Process.js │ │ │ │ │ ├── ProcessManager.js │ │ │ │ │ ├── Status.js │ │ │ │ │ ├── Step.js │ │ │ │ │ ├── Task.js │ │ │ │ │ ├── expectedAnswersMixin.js │ │ │ │ │ └── stateManagerMixin.js │ │ │ ├── manifestSources │ │ │ │ ├── CSVSpreadsheet.js │ │ │ │ ├── Git.js │ │ │ │ ├── GoogleFonts.js │ │ │ │ └── _Source.js │ │ │ └── util │ │ │ │ ├── AsyncQueue.js │ │ │ │ ├── DispatcherProcessManagerClient.js │ │ │ │ ├── GitHubAuthClient.js │ │ │ │ ├── GitHubOperationsClient.js │ │ │ │ ├── IOOperations.js │ │ │ │ ├── InitWorkersClient.js │ │ │ │ ├── Logging.js │ │ │ │ ├── ManifestClient.js │ │ │ │ ├── ManifestServer.js │ │ │ │ ├── ProcessManagerClient.js │ │ │ │ ├── ProtobufAnyHandler.js │ │ │ │ ├── ReportsClient.js │ │ │ │ ├── StorageClient.js │ │ │ │ ├── StorageServers.js │ │ │ │ ├── getMetadataPb.js │ │ │ │ ├── getSetup.js │ │ │ │ └── nodeCallback2Promise.js │ │ └── package.json │ ├── protocolbuffers │ │ ├── messages.proto │ │ └── shared.proto │ ├── python │ │ ├── Dockerfile │ │ ├── __init__.py │ │ ├── debug_run.py │ │ ├── debug_vollkorn │ │ │ ├── after │ │ │ │ ├── Vollkorn-Black.ttf │ │ │ │ ├── Vollkorn-BlackItalic.ttf │ │ │ │ ├── Vollkorn-Bold.ttf │ │ │ │ ├── Vollkorn-BoldItalic.ttf │ │ │ │ ├── Vollkorn-ExtraBold.ttf │ │ │ │ ├── Vollkorn-ExtraBoldItalic.ttf │ │ │ │ ├── Vollkorn-Italic.ttf │ │ │ │ ├── Vollkorn-Medium.ttf │ │ │ │ ├── Vollkorn-MediumItalic.ttf │ │ │ │ ├── Vollkorn-Regular.ttf │ │ │ │ ├── Vollkorn-SemiBold.ttf │ │ │ │ ├── Vollkorn-SemiBoldItalic.ttf │ │ │ │ ├── VollkornSC-Black.ttf │ │ │ │ ├── VollkornSC-Bold.ttf │ │ │ │ ├── VollkornSC-ExtraBold.ttf │ │ │ │ ├── VollkornSC-Medium.ttf │ │ │ │ ├── VollkornSC-Regular.ttf │ │ │ │ ├── VollkornSC-SemiBold.ttf │ │ │ │ └── vf │ │ │ │ │ ├── Vollkorn-Italic-VF.ttf │ │ │ │ │ └── Vollkorn-Roman-VF.ttf │ │ │ └── before │ │ │ │ ├── OFL.txt │ │ │ │ ├── Vollkorn-Black.ttf │ │ │ │ ├── Vollkorn-BlackItalic.ttf │ │ │ │ ├── Vollkorn-Bold.ttf │ │ │ │ ├── Vollkorn-BoldItalic.ttf │ │ │ │ ├── Vollkorn-Italic.ttf │ │ │ │ ├── Vollkorn-Regular.ttf │ │ │ │ ├── Vollkorn-SemiBold.ttf │ │ │ │ └── Vollkorn-SemiBoldItalic.ttf │ │ ├── protocolbuffers │ │ │ ├── __init__.py │ │ │ ├── fonts_public_pb2.py │ │ │ ├── fonts_public_pb2_grpc.py │ │ │ ├── messages_pb2.py │ │ │ ├── messages_pb2_grpc.py │ │ │ ├── shared_pb2.py │ │ │ └── shared_pb2_grpc.py │ │ ├── requirements.txt │ │ ├── worker-launcher.py │ │ └── worker │ │ │ ├── __init__.py │ │ │ ├── diff_tools_shared.py │ │ │ ├── diffbrowsers.py │ │ │ ├── diffenator.py │ │ │ ├── fontbakery.py │ │ │ ├── storageclient.py │ │ │ └── worker_base.py │ └── update_protobufs.sh └── rethinkdb │ ├── Dockerfile │ ├── Makefile │ ├── files │ └── run.sh │ └── rethinkdb-probe │ ├── Dockerfile.build │ ├── build-probe.sh │ ├── probe.go │ └── rethinkdb-probe ├── docs ├── DEPLOYLOG.md ├── minikube-help.md ├── nodes-diagram-v2.inkscape.svg ├── nodes-diagram-v2.pdf └── the-report-format.md ├── kubernetes ├── gcloud-fontbakery-api.yaml ├── gcloud-fontbakery-dispatcher.yaml ├── gcloud-fontbakery-github-auth.yaml ├── gcloud-fontbakery-github-operations.yaml ├── gcloud-fontbakery-init-workers.yaml ├── gcloud-fontbakery-manifest-csvupstream.yaml ├── gcloud-fontbakery-manifest-gfapi.yaml ├── gcloud-fontbakery-manifest-githubgf.yaml ├── gcloud-fontbakery-manifest-main.yaml ├── gcloud-fontbakery-reports.yaml ├── gcloud-fontbakery-storage-cache.yaml ├── gcloud-fontbakery-storage-persistence.yaml ├── gcloud-fontbakery-worker.yaml ├── gcloud-rabbitmq.yaml ├── gcloud-rethinkdb.yaml ├── minikube-fontbakery-api.yaml ├── minikube-fontbakery-dispatcher.yaml ├── minikube-fontbakery-github-auth.yaml ├── minikube-fontbakery-github-operations.yaml ├── minikube-fontbakery-init-workers.yaml ├── minikube-fontbakery-manifest-csvupstream.yaml ├── minikube-fontbakery-manifest-gfapi.yaml ├── minikube-fontbakery-manifest-githubgf.yaml ├── minikube-fontbakery-manifest-main.yaml ├── minikube-fontbakery-reports.yaml ├── minikube-fontbakery-storage-cache.yaml ├── minikube-fontbakery-storage-persistence.yaml ├── minikube-fontbakery-worker.yaml ├── minikube-rabbitmq.yaml └── minikube-rethinkdb.yaml └── skaffold.yaml /AUTHORS.txt: -------------------------------------------------------------------------------- 1 | # This is the official list of Font Bakery authors for copyright purposes. 2 | # This file is distinct from the CONTRIBUTORS files. 3 | # See the latter for an explanation. 4 | 5 | # Names should be added to this file as: 6 | # Name or Organization 7 | # The email address is not required for organizations. 8 | 9 | Google Inc. 10 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | The project is run on Github, in the [typical](http://producingoss.com) free software way. 2 | We'd love to accept your patches and contributions to this project. 3 | There is just one thing you need to do first. 4 | 5 | ## Google CLA 6 | 7 | Contributions to Google projects, even unofficial ones like this, must be accompanied by a Contributor License Agreement. 8 | This is not a copyright assignment, it simply gives Google permission to use and redistribute your contributions as part of the project. 9 | 10 | If you are an individual writing original source code and you're sure you own the rights, then you'll need to sign an 11 | [individual CLA](https://developers.google.com/open-source/cla/individual). 12 | 13 | If you work for a company that wants to allow you to contribute your work, then you'll need to sign a [corporate CLA](https://developers.google.com/open-source/cla/corporate). 14 | 15 | You generally only need to submit a CLA once, so if you've already submitted one for a different project, you probably don't need to do it again. 16 | 17 | After your contribution is included, you will be listed in CONTRIBUTORS.txt and/or AUTHORS.txt: CONTRIBUTORS is the official list of people who can contribute (and typically have contributed) code to this repository, while the AUTHORS file lists the copyright holders. 18 | 19 | ## How to Contribute 20 | 21 | Our production vision and high level requirements are explained in the [fontbakery/BRIEF.md](https://github.com/googlefonts/fontbakery/blob/main/BRIEF.md) 22 | 23 | We use Github's Issue Tracker to report and track bugs, map out future improvements, set priorities and assign people to them. 24 | 25 | We only assign an issue to someone when they are going to work on that issue soon. 26 | 27 | We tag all issues with a priority tag, P0 for "urgent" through P5 for "someday-maybe" 28 | 29 | If you find a bug or have an idea, please create a new issue and we'll be happy to work with you on it! 30 | 31 | -------------------------------------------------------------------------------- /CONTRIBUTORS.txt: -------------------------------------------------------------------------------- 1 | # This is the official list of people who can contribute 2 | # (and typically have contributed) code to this repository. 3 | # The AUTHORS file lists the copyright holders; this file 4 | # lists people. For example, Google employees are listed here 5 | # but not in AUTHORS, because Google holds the copyright. 6 | # 7 | # Names should be added to this file only after verifying that 8 | # the individual or the individual's organization has agreed to 9 | # the appropriate Contributor License Agreement, found here: 10 | # 11 | # http://code.google.com/legal/individual-cla-v1.0.html 12 | # http://code.google.com/legal/corporate-cla-v1.0.html 13 | # 14 | # The agreement for individuals can be filled out on the web. 15 | # 16 | # When adding J Random Contributor's name to this file, 17 | # either J's name or J's organization's name should be 18 | # added to the AUTHORS file, depending on whether the 19 | # individual or corporate CLA was used. 20 | # 21 | # Names should be added to this file like so: 22 | # Name 23 | # 24 | # Please keep the list sorted. 25 | # (first name; alphabetical order) 26 | 27 | Dave Crossland 28 | Felipe Sanches 29 | Lasse Fister 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![License: Apache 2.0](https://img.shields.io/badge/License-Apache%202.0-brightgreen.svg)](https://github.com/googlefonts/fontbakery/blob/main/LICENSE.txt) 2 | 3 | Font Bakery-Dashboard was a web application to make [Font Bakery](https://github.com/googlefonts/fontbakery) (a command-line font project checker) available easily to everyone. 4 | 5 | A Live Demo used to be hosted at fontbakery.graphicore.de but it is no longer available. 6 | 7 | The project was halted at the end of 2019. 8 | 9 | #### Your Cloud of Choice 10 | 11 | We ran Fontbakery Dashboard on the Google Cloud and for development on Minikube, but it really is your choice where to host your dashboard. 12 | Kubernetes is supported by many more providers (like Amazon Web Services, Microsoft Azure, etc) and you can learn more about this in the [kubernetes docs](https://kubernetes.io/docs/setup/pick-right-solution). 13 | -------------------------------------------------------------------------------- /containers/base/.dockerignore: -------------------------------------------------------------------------------- 1 | # we exclude these as a way to make sure our package.json and bower.json 2 | # are defining the right dependencies the alternative would be to rely 3 | # on the state of the developer machine that builds the image, which is 4 | # a moving target. 5 | **/node_modules 6 | **/bower_components 7 | # used in local development 8 | **/many_csvfontsgit 9 | -------------------------------------------------------------------------------- /containers/base/.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | devvenv 3 | protocolbuffers/tools 4 | protocolbuffers/fonts_public.proto 5 | -------------------------------------------------------------------------------- /containers/base/dev-requirements.txt: -------------------------------------------------------------------------------- 1 | protobuf 2 | grpcio 3 | grpcio-tools 4 | -------------------------------------------------------------------------------- /containers/base/javascript/.bowerrc: -------------------------------------------------------------------------------- 1 | { 2 | "directory" : "browser/js/bower_components" 3 | } 4 | -------------------------------------------------------------------------------- /containers/base/javascript/.dockerignore: -------------------------------------------------------------------------------- 1 | git-repositories 2 | -------------------------------------------------------------------------------- /containers/base/javascript/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | node_modules 3 | bower_components 4 | -------------------------------------------------------------------------------- /containers/base/javascript/.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "laxcomma": true, 3 | "laxbreak": true, 4 | "undef": true, 5 | "unused": true, 6 | "predef": [ "define", "require", "Promise"], 7 | "sub": true 8 | } 9 | -------------------------------------------------------------------------------- /containers/base/javascript/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:13.13.0-buster 2 | 3 | 4 | RUN touch /tmp/cache_spoof_16 5 | 6 | RUN apt-get update && apt-get upgrade -y \ 7 | && apt-get install -y software-properties-common 8 | 9 | # mentioned in https://github.com/nodegit/nodegit 10 | RUN apt-get update \ 11 | && apt-get install -y libssl-dev libpcre2-posix0 libkrb5-3 \ 12 | libk5crypto3 libcomerr2 libpcre2-dev libkrb5-dev 13 | 14 | 15 | RUN npm install -g bower browserify requirejs; 16 | 17 | ADD ./.bowerrc ./bower.json ./package.json /var/javascript/ 18 | 19 | RUN touch /tmp/cache_spoof_2 20 | ADD ./generated_modules /var/javascript/generated_modules 21 | 22 | RUN touch /tmp/cache_spoof_2 23 | 24 | # will also invoke bower install etc. 25 | # --production: don't install dev-dependencies 26 | RUN cd /var/javascript && npm install --unsafe-perm --production 27 | 28 | 29 | # It seems to be most robust to read our METADATA.pb files using python 30 | # that's the only reason for this dependency, used in manifestSources 31 | # via the module /node/util/getMetadataPb 32 | RUN apt-get update \ 33 | && apt-get install -y build-essential cmake python3-dev git; 34 | RUN curl -o /tmp/get-pip.py https://bootstrap.pypa.io/get-pip.py; python3 /tmp/get-pip.py; 35 | RUN pip install --upgrade pip 36 | 37 | # opentype-sanitizer needs ninja which did not build without these: 38 | # related: https://github.com/scikit-build/ninja-python-distributions/issues/22 39 | RUN pip install scikit-build 40 | # help ninja build itself 41 | ENV CXXFLAGS="-D_BSD_SOURCE" 42 | # build ninja 43 | RUN pip install ninja 44 | 45 | 46 | RUN git clone --depth 1 -b main https://github.com/googlefonts/gftools.git /var/gftools;\ 47 | pip install /var/gftools; 48 | 49 | RUN cd /var/javascript \ 50 | && bower install --allow-root \ 51 | && browserify ./browser/js/bower_components/jiff/jiff.js -s jiff \ 52 | -o ./browser/js/bower_components/jiff/jiff.umd.js && browserify \ 53 | ./browser/js/bower_components/jiff/lib/jsonPointer -s jsonPointer \ 54 | -o ./browser/js/bower_components/jiff/jsonPointer.umd.js \ 55 | && cp -r ./generated_modules/protocolbuffers ./node_modules/protocolbuffers \ 56 | && mkdir -p ./browser/js/bower_components/protocolbuffers \ 57 | && browserify ./node_modules/protocolbuffers/shared_pb.js -s protocolbuffers/shared_pb \ 58 | -o ./browser/js/bower_components/protocolbuffers/shared_pb.js; 59 | 60 | # TODO: install with bower when: 61 | # https://github.com/drudru/ansi_up/issues/39 is resolved 62 | RUN mkdir -p /var/javascript/browser/js/bower_components/ansi_up/ \ 63 | && curl -o /var/javascript/browser/js/bower_components/ansi_up/ansi_up.js \ 64 | https://raw.githubusercontent.com/drudru/ansi_up/master/ansi_up.js 65 | 66 | # In the minikube development version it can be interesting to preload 67 | # the image with a current version of the github google/fonts repository. 68 | # If so in this directory (containers/base/javascript) do: 69 | # $ git clone --bare https://github.com/google/fonts.git fontsgit 70 | # Otherwise, if image file size is an issue: 71 | # $ mkdir fontsgit 72 | # Would be nice to have the ADD optional ... 73 | # ADD ./fontsgit /var/git-repositories/github.com_google_fonts.git 74 | 75 | # ADD ./fontsgit /var/fontsgit 76 | # RUN ln -s /var/fontsgit /var/git-repositories 77 | 78 | # This way we only need to rebuild the last intermediate container when 79 | # code changed, especially the npm install can be skipped! 80 | ADD ./browser /var/javascript/browser 81 | ADD ./node /var/javascript/node 82 | -------------------------------------------------------------------------------- /containers/base/javascript/README.md: -------------------------------------------------------------------------------- 1 | TODO: Document the server/concepts 2 | 3 | 4 | # Install: 5 | 6 | ``` 7 | $ npm install 8 | ``` 9 | 10 | # start the server 11 | 12 | ``` 13 | $ npm start 14 | ``` 15 | -------------------------------------------------------------------------------- /containers/base/javascript/THOUGHTS.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | a client connects 4 | create the rethink db query 5 | with all initial true, so we don't have race conditions 6 | associate client with query 7 | 8 | 9 | a query answers 10 | push to the client who's connected to the query 11 | 12 | * Problems SocketIO seems not to support to push to specific clients on 13 | Python (probably wrong) 14 | * rethinkdb change is blocking, we can't run many of these using not Threads 15 | 16 | ws = 17 | g = gevent.Greenlet.spawn(read_feed).get() 18 | 19 | 20 | if a document has ended, it would be wise to close the websocket after 21 | sending the data. 22 | It would be cooler to hang up server side than client side! 23 | 24 | 25 | a client disconnects 26 | destroy the query 27 | 28 | a feed ends 29 | send that to the client (and hang up?) 30 | 31 | 32 | The plan dies because: each test needs all files, otherwise we can't run family tests successfully :-( 33 | Also, family fonts themselves have different rules than single fonts. 34 | 35 | Tough, as a preparation for a post-refactoring client, this would still be a good start. 36 | We can just send at the end of a fb run X-different messages to the rethinkdb client, 37 | having all the infrastructure in place for more complex runs. 38 | 39 | 40 | -------------------------------------------------------------------------------- /containers/base/javascript/bower.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "fontbakery-drag-and-drop", 3 | "version": "0.0.0", 4 | "authors": [ 5 | "Felipe Sanches ", 6 | "Lasse Fister " 7 | ], 8 | "moduleType": [ 9 | "amd" 10 | ], 11 | "license": "APACHE2", 12 | "homepage": "https://github.com/googlefonts/fontbakery", 13 | "private": true, 14 | "ignore": [ 15 | "**/.*", 16 | "node_modules", 17 | "bower_components", 18 | "test", 19 | "tests" 20 | ], 21 | "dependencies": { 22 | "requirejs": "~2.3.2", 23 | "marked": "~0.3.4", 24 | "jiff": "~0.7.3", 25 | "ansi_up": "https://github.com/drudru/ansi_up.git" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /containers/base/javascript/browser/img/drop_file_icon.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /containers/base/javascript/browser/img/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/javascript/browser/img/favicon.ico -------------------------------------------------------------------------------- /containers/base/javascript/browser/img/fb_logo.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /containers/base/javascript/browser/img/font_icon.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /containers/base/javascript/browser/img/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/javascript/browser/img/logo.png -------------------------------------------------------------------------------- /containers/base/javascript/browser/img/logo_200px.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/javascript/browser/img/logo_200px.png -------------------------------------------------------------------------------- /containers/base/javascript/browser/img/logo_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/javascript/browser/img/logo_icon.png -------------------------------------------------------------------------------- /containers/base/javascript/browser/img/oven_icon.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /containers/base/javascript/browser/img/report_icon.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /containers/base/javascript/browser/js/CollectionController.js: -------------------------------------------------------------------------------- 1 | define([ 2 | 'dom-tool' 3 | , 'xhr' 4 | ], function( 5 | dom 6 | , xhr 7 | ) { 8 | "use strict"; 9 | /*global console*/ 10 | 11 | function CollectionController(container) { 12 | this.container = container; 13 | this._getReportLinks(); 14 | } 15 | var _p = CollectionController.prototype; 16 | 17 | _p._insertReportLinks = function(items) { 18 | var i, l, item, a, li, marker; 19 | 20 | items.forEach(function(item){ 21 | item.date = new Date(item.date); 22 | }); 23 | items.sort(function(a, b) { 24 | return a.date - b.date; 25 | }); 26 | items.reverse(); 27 | 28 | marker = dom.getMarkerComment(this.container, 'insert: link'); 29 | for(i=0,l=items.length;i=0;i--) { 63 | if(subscribers[i][2] === marker) 64 | subscribers.splice(i, 1); 65 | } 66 | }; 67 | 68 | // Just a thought: using promises as a return of _publish could 69 | // establish a nice back channel here! for whatever it's worth ;-) 70 | // In general, though, I would expect publish to be **sync**. 71 | _p.publish = function(topic) { 72 | var message = [], i, l 73 | , topicData = this._getTopic(topic) 74 | , subscribers = topicData.subscribers 75 | ; 76 | for(i=1,l=arguments.length;i= 0 && 46 | rect.left >= 0 && 47 | rect.bottom <= (window.innerHeight || document.documentElement.clientHeight) && /*or $(window).height() */ 48 | rect.right <= (window.innerWidth || document.documentElement.clientWidth) /*or $(window).width() */ 49 | ); 50 | } 51 | 52 | _p._onViewportChange = function(e) { 53 | // jshint unused: vars 54 | var lastItem = this._lastItem 55 | , lastItemElement = this._pageItems[this._pageItems.length-1] 56 | , url 57 | ; 58 | if(!lastItemElement) 59 | return; 60 | if(!isElementInViewport(lastItemElement)) 61 | return; 62 | if(this._receivedLastPage) 63 | return; 64 | // The url for a previous page, e.g. could be used to poll 65 | // if there are newer reports. We don't do this now. 66 | // ['/status-reports', firstItem.reported, firstItem.id, 'true'].join('/')] 67 | url = ['/status-reports', lastItem.reported, lastItem.id, 'false'].join('/'); 68 | if(this._requestedUrls.has(url)) 69 | return; 70 | this._requestedUrls.add(url); 71 | this._getReportLinks(url); 72 | }; 73 | 74 | _p._receiveReportLinks = function(items) { 75 | var i, l, item, a, li, marker; 76 | if(!items.length) { 77 | // CAUTION: this only works when we use this function fpr 78 | // next-pages, when receiving previous pages/polling for newer 79 | // we'll often get empty items and this needs to change, 80 | this._receivedLastPage = true; 81 | return; 82 | } 83 | this._lastItem = items[items.length-1]; 84 | marker = dom.getMarkerComment(this.container, 'insert: status-report-link'); 85 | // Items are sorted descending, from new to old, thus the first 86 | // pages shows the latest entries. 87 | for(i=0,l=items.length;i 'object' 21 | if(a === null || b === null) 22 | return false; 23 | if(typeof a === 'object') { 24 | if(typeof b !== 'object') 25 | return false; 26 | for(k in a) { 27 | if(!(k in b)) 28 | return false; 29 | if(!isEqual(a[k], b[k])) 30 | return false; 31 | } 32 | return true; 33 | } 34 | return false; 35 | }; 36 | return isEqual; 37 | }); 38 | -------------------------------------------------------------------------------- /containers/base/javascript/browser/js/xhr.js: -------------------------------------------------------------------------------- 1 | define([], function() { 2 | "use strict"; 3 | /*global XMLHttpRequest, Promise*/ 4 | 5 | function _sendXHR(xhr, data) { 6 | return new Promise(function(resolve, reject){ 7 | xhr.send(data); 8 | xhr.onreadystatechange = function () { 9 | if(xhr.readyState !== XMLHttpRequest.DONE) 10 | return; 11 | if(xhr.status !== 200) 12 | reject(new Error('XHR not OK; code ' + xhr.status 13 | + '; ' + xhr.statusText)); 14 | else 15 | resolve(xhr.response); 16 | }; 17 | }); 18 | } 19 | 20 | function getJSON(adress, data) { 21 | var xhr = new XMLHttpRequest(); 22 | xhr.open('GET', adress); 23 | xhr.setRequestHeader('Content-Type', 'application/json'); 24 | xhr.responseType = 'json'; 25 | return _sendXHR(xhr, data); 26 | } 27 | 28 | return { 29 | getJSON: getJSON 30 | }; 31 | }); 32 | -------------------------------------------------------------------------------- /containers/base/javascript/generated_modules/protocolbuffers/fonts_public_grpc_pb.js: -------------------------------------------------------------------------------- 1 | // GENERATED CODE -- NO SERVICES IN PROTO -------------------------------------------------------------------------------- /containers/base/javascript/generated_modules/protocolbuffers/shared_grpc_pb.js: -------------------------------------------------------------------------------- 1 | // GENERATED CODE -- NO SERVICES IN PROTO -------------------------------------------------------------------------------- /containers/base/javascript/node/CacheServer.js: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | "use strict"; 3 | // this is expected to run in nodejs 4 | /* global require, module */ 5 | /* jshint esnext:true */ 6 | 7 | const getSetup = require('./util/getSetup').getSetup 8 | , { CacheServer } = require('./util/StorageServers') 9 | ; 10 | 11 | /** 12 | What we want: 13 | 14 | A) POST Files: returns an identifier where to GET Files 15 | B) GET Files: 16 | C) PURGE Files 17 | 18 | use gRPC 19 | 20 | Files is one "Family" of TTF files and metadata. Ideally we don't even look into the serialized message 21 | 22 | There's probably a client-id in this message, so that the client can identify the answer 23 | I guess gRPC has something buildin for us ;-) 24 | 25 | Post Files can actually post a stream of files, which will conclude a whole collection 26 | The answer then is a stream of identifiers that can be used to GET or PURGE the files 27 | 28 | If you post the same Files twice, you got to PURGE the same file twice (unless force===true) 29 | 30 | TODO: let's say a job died and is never going to PURGE. We need some 31 | automatic cache invalidation (time based?), maybe pick something from 32 | here: https://en.wikipedia.org/wiki/Cache_replacement_policies 33 | 34 | TODO: paralell cache instances, that talk to each other for replcation 35 | if speed or memory size becomes a problem? 36 | 37 | TODO: some form of persistence, if the pod dies, so that the cache still 38 | has its state after being up again. 39 | */ 40 | 41 | if (typeof require != 'undefined' && require.main==module) { 42 | var setup = getSetup(), cacheServer, port=50051 43 | // Keep entries for 24 hours, this timeout ensures the CacheServer 44 | // won't run out of memory BUT it expects that the client will 45 | // consume the data in a timely manner. So, if we don't have enough 46 | // workers for a big job, eventually we'll have failing jobs because 47 | // the cache cleaned up itself ... 48 | // To avoid this cleaning up, don't define dataItemTimeOutMinutes 49 | , dataItemTimeOutMinutes = 24 * 60 50 | ; 51 | 52 | for(let i=0,l=process.argv.length;i= 0) // not NaN or negative 56 | port = foundPort; 57 | break; 58 | } 59 | } 60 | setup.logging.info('Init server, port: '+ port +' ...'); 61 | setup.logging.log('Loglevel', setup.logging.loglevel); 62 | cacheServer = new CacheServer(setup.logging, port, dataItemTimeOutMinutes); 63 | cacheServer.serve(); 64 | } 65 | -------------------------------------------------------------------------------- /containers/base/javascript/node/ManifestMaster.js: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | "use strict"; 3 | // this is expected to run in nodejs 4 | /* global require */ 5 | /* jshint esnext:true */ 6 | 7 | const { getSetup } = require('./util/getSetup') 8 | , { IOOperations } = require('./util/IOOperations') 9 | , messages_pb = require('protocolbuffers/messages_pb') 10 | , { StorageClient } = require('./util/StorageClient') 11 | , { InitWorkersClient } = require('./util/InitWorkersClient') 12 | ; 13 | 14 | /** 15 | * The ManifestMain receives requests for tests from the Manifests 16 | * via a queue. 17 | * It then decides whether to execute the test run or if it can just 18 | * shortcut the execution, because an identical test already exists. 19 | * 20 | * A test is identical when the testing environment (fontbakery etc. 21 | * see ENVIRONMENT_VERSION) and it's data (test_data_hash) are equal. 22 | * 23 | * Thus the familytest table has a secondary key: [environment_version test_data_hash] 24 | * 25 | * On an incoming request we query the familytest table for the 26 | * [environment_id test_data_hash] key. if an entry comes up, we can use 27 | * the familytests_id otherwise, we create a new entry then a) dispatch to 28 | * the worker-distributor 29 | * b) use the familytests_id to create an entry in the collections table 30 | * 31 | * This skips the test if it already exists! 32 | * 33 | * the collectiontests table: 34 | * 35 | * collection_id | family_name | familytests_id | date (sorting) | family_metadata 36 | * 37 | * Don't do consecutive entries that are the same 38 | * collection_id | family_name | familytests_id 39 | * the last entry must not be like that, if there's somewhere another entry 40 | * like this, we're probably watching at a rollback, which is fine. 41 | * 42 | * get collection_id | family_name 43 | * if there is a an entry and familytests_id == new familytest_id 44 | * dont' create the entry 45 | * else 46 | * create the full entry 47 | */ 48 | function ManifestMain(logging, setup) { 49 | this._log = logging; 50 | this._io = new IOOperations(logging, setup.db, setup.amqp); 51 | this._manifestMainJobQueueName = 'fontbakery-manifest-main-jobs'; 52 | this._cache = new StorageClient(logging, setup.cache.host, setup.cache.port); 53 | this._initWorkers = new InitWorkersClient(logging, setup.initWorkers.host, setup.initWorkers.port); 54 | 55 | // Start serving when the database and rabbitmq queue is ready 56 | Promise.all([ 57 | this._io.init() 58 | , this._cache.waitForReady() 59 | , this._initWorkers.waitForReady() 60 | ]) 61 | //.then(resources => { 62 | // amqp = resources[0][1]; 63 | //}) 64 | .then(this._listen.bind(this)) 65 | .then( 66 | ()=>this._log.info('Server ready!') 67 | , err => { 68 | this._log.error('Can\'t initialize server.', err); 69 | process.exit(1); 70 | }); 71 | } 72 | 73 | var _p = ManifestMain.prototype; 74 | 75 | _p._listen = function() { 76 | return this._io.queueListen( 77 | this._manifestMainJobQueueName, this._consumeQueue.bind(this)); 78 | }; 79 | 80 | _p._getCollectionFamilyJob = function(messageContent) { 81 | var arr = new Uint8Array(Buffer.from(messageContent)) 82 | , job = messages_pb.CollectionFamilyJob.deserializeBinary(arr) 83 | ; 84 | // this._log.debug('collectionFamilyJob', job.toObject()); 85 | return job; 86 | }; 87 | 88 | /* 89 | * On an incoming request we 90 | * a) query the familytest table for the [environment_id test_data_hash] key. 91 | * if an entry comes up, we can use the familytest id (docid) 92 | * else we create a new entry and dispatch to the worker-distributor 93 | * then use the familytest id 94 | * 95 | * b) use the familytests_id to create an entry in the collections table 96 | */ 97 | _p._consumeQueue = function(message) { 98 | var job = this._getCollectionFamilyJob(message.content) 99 | , cacheKey = job.getCacheKey() 100 | ; 101 | 102 | return this._initWorkers.initialize('fontbakery', cacheKey) 103 | .then(familyJob=>{ 104 | var docid = familyJob.getDocid(); 105 | return this._createCollectionEntry(job, docid); 106 | }) 107 | .then(() => this._io.ackQueueMessage(message)) 108 | .catch(err=>this._log.error(err)) // die now? 109 | ; 110 | }; 111 | 112 | 113 | /** 114 | * These are only created and never changed again. 115 | * we try to not create consecutive entries 116 | * of equal [collection_id | family_name | familytests_id], but this 117 | * is not quaranteed du to race conditions 118 | * 119 | * We can thus monitor changes in here and then update a latest/live view 120 | * when new entries are created. 121 | */ 122 | _p._createCollectionEntry = function(job, familytests_id) { 123 | var collection_id = job.getCollectionid() 124 | , family_name = job.getFamilyName() 125 | , metadata = job.getMetadata() 126 | , doc = { 127 | collection_id: collection_id 128 | , family_name: family_name 129 | , familytests_id: familytests_id 130 | , date: job.getDate().toDate() 131 | , metadata: metadata ? JSON.parse(metadata) : {} 132 | } 133 | ; 134 | 135 | // Don't do consecutive entries that are the same 136 | // [collection_id | family_name | familytests_id] 137 | return this._io.getLatesCollectionEntry(collection_id, family_name) 138 | // => collectiontests_id; 139 | .then(collectionDoc => { 140 | if(collectionDoc && collectionDoc.familytests_id === familytests_id) { 141 | // don't insert, it's equal 142 | this._log.debug('collectionDoc is already latest:' 143 | , collection_id, family_name, familytests_id); 144 | return collectionDoc.id; 145 | } 146 | return this._io.insertDoc('collection', doc) 147 | .then(response=>response.generated_keys[0]);// => collectiontests_id; 148 | }) 149 | ; 150 | }; 151 | 152 | if (typeof require != 'undefined' && require.main==module) { 153 | var setup = getSetup(); 154 | setup.logging.log('Loglevel', setup.logging.loglevel); 155 | new ManifestMain(setup.logging, { 156 | // passing these explicitly to document the dependencies 157 | db: setup.db 158 | , amqp: setup.amqp 159 | , cache: setup.cache 160 | , initWorkers: setup.initWorkers 161 | }); 162 | } 163 | -------------------------------------------------------------------------------- /containers/base/javascript/node/PersistenceServer.js: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | "use strict"; 3 | // this is expected to run in nodejs 4 | /* global require, module */ 5 | /* jshint esnext:true */ 6 | 7 | const getSetup = require('./util/getSetup').getSetup 8 | , { PersistenceServer } = require('./util/StorageServers') 9 | ; 10 | 11 | if (typeof require != 'undefined' && require.main==module) { 12 | var setup = getSetup(), persistenceServer, port=50051 13 | , dataItemTimeOutMinutes = Infinity // never garbage collect 14 | , dataDirDefault = '/tmp/persistence_server' 15 | , dataDir = null 16 | ; 17 | 18 | for(let i=0,l=process.argv.length;i= 0) // not NaN or negative 22 | port = foundPort; 23 | i++; 24 | } 25 | if(process.argv[i] === '-d' && i+1throw new Error())` 52 | // It also seems to result in a cancel, as well as when the 53 | // client just shuts down. There is not really a way to send 54 | // errors to the server as it seems. 55 | 56 | 57 | // TODO: need to keep call in `subscriptions` structure 58 | call.on('error', error=>{ 59 | this._log.error('on ERROR: subscribeCall('+type+'):', error); 60 | unsubscribe(); 61 | }); 62 | call.on('cancelled', ()=>{ 63 | // hmm somehow is called after the `call.on('error',...)` handler, 64 | // at least when triggered by `call.destroy(new Error(...))` 65 | // seems like this is called always when the stream is ended 66 | // we should be careful with not trying to double-cleanup here 67 | // if the client cancels, there's no error though! 68 | this._log.debug('on CANCELLED: subscribeCall('+type+')'); 69 | unsubscribe(); 70 | }); 71 | 72 | call.on('finish', ()=>{ 73 | this._log.debug('on FINISH: subscribeCall('+type+')'); 74 | unsubscribe(); 75 | }); 76 | }; 77 | 78 | _p.subscribeProcess = function(call) { 79 | var processQuery = call.request 80 | , unsubscribe = ()=> { 81 | if(!timeout) // marker if there is an active subscription/call 82 | return; 83 | // End the subscription and delete the call object. 84 | // Do this only once, but, `unsubscribe` may be called more than 85 | // once, e.g. on `call.destroy` via FINISH, CANCELLED and ERROR. 86 | this._log.info('... UNSUBSCRIBE'); 87 | clearInterval(timeout); 88 | timeout = null; 89 | } 90 | ; 91 | 92 | this._log.info('processQuery subscribing to', processQuery.getProcessId()); 93 | this._subscribeCall('process', call, unsubscribe); 94 | 95 | var counter = 0, maxIterations = Infinity 96 | , timeout = setInterval(()=>{ 97 | this._log.debug('subscribeProcess call.write counter:', counter); 98 | var processState = new ProcessState(); 99 | processState.setProcessId(new Date().toISOString()); 100 | 101 | counter++; 102 | if(counter === maxIterations) { 103 | //call.destroy(new Error('Just a random server fuckup.')); 104 | //clearInterval(timeout); 105 | call.end(); 106 | } 107 | else 108 | call.write(processState); 109 | 110 | }, 1000); 111 | }; 112 | 113 | _p.subscribeProcessList = function(call) { 114 | var processListQuery = call.request 115 | , unsubscribe = ()=> { 116 | if(!timeout) // marker if there is an active subscription/call 117 | return; 118 | // End the subscription and delete the call object. 119 | // Do this only once, but, `unsubscribe` may be called more than 120 | // once, e.g. on `call.destroy` via FINISH, CANCELLED and ERROR. 121 | this._log.info('... UNSUBSCRIBE'); 122 | clearInterval(timeout); 123 | timeout = null; 124 | } 125 | ; 126 | 127 | this._log.info('processQuery subscribing to', processListQuery.getQuery()); 128 | this._subscribeCall('process', call, unsubscribe); 129 | 130 | var counter = 0, maxIterations = Infinity 131 | , timeout = setInterval(()=>{ 132 | this._log.debug('subscribeProcessList call.write counter:', counter); 133 | 134 | var processList = new ProcessList(); 135 | for(let i=0,l=3;i>>' + new Date().toISOString()); 139 | processList.addProcesses(processListItem); 140 | } 141 | 142 | counter++; 143 | if(counter === maxIterations) { 144 | //call.destroy(new Error('Just a random server fuckup.')); 145 | //clearInterval(timeout); 146 | call.end(); 147 | } 148 | else 149 | call.write(processList); 150 | 151 | }, 1000); 152 | }; 153 | 154 | _p.execute = function(processCommand) { 155 | TODO(processCommand); 156 | return new Empty(); 157 | }; 158 | 159 | if (typeof require != 'undefined' && require.main==module) { 160 | var { getSetup } = require('../util/getSetup') 161 | , setup = getSetup(), processManager, port=50051; 162 | 163 | for(let i=0,l=process.argv.length;i= 0) // not NaN or negative 167 | port = foundPort; 168 | break; 169 | } 170 | } 171 | setup.logging.info('Init server, port: '+ port +' ...'); 172 | setup.logging.log('Loglevel', setup.logging.loglevel); 173 | // Combine ProcessManager and the process definition:FamilyPRDispatcherProcess. 174 | processManager = new DummyProcessManager(setup.logging, port); 175 | processManager.serve(); 176 | } 177 | -------------------------------------------------------------------------------- /containers/base/javascript/node/dispatcher/framework/GenericProcess.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | /* jshint esnext:true, node:true*/ 3 | 4 | const { Task } = require('./Task') 5 | , { Step } = require('./Step') 6 | , { Process } = require('./Process') 7 | , {mixin: stateManagerMixin} = require('./stateManagerMixin') 8 | ; 9 | 10 | 11 | /** 12 | * This is really not elegant, but I think it will do it's 13 | * job just fine. 14 | */ 15 | function manipulateStateManagerValidation(state, ignore) { 16 | // jshint validthis: true 17 | var forgivingStateDefinitions = {} 18 | , stateDefKeys = new Set() 19 | , isAlwaysValid = ()=>[true, null] 20 | , isTrue = ()=>true 21 | , isFalse = ()=>false 22 | // EXPECTED and present 23 | , acceptingDefinition = { 24 | init: ()=>null 25 | , serialize: state=>state 26 | , load: state=>state 27 | } 28 | ; 29 | for(let [key, definition] of this._stateDefEntries()) { 30 | if(ignore && ignore.has(key)) 31 | continue; 32 | stateDefKeys.add(key); 33 | let newDef = Object.create(definition); 34 | if('validate' in definition) 35 | newDef.validate = isAlwaysValid; 36 | 37 | if(!(key in state)) 38 | newDef.isExpected = isFalse; 39 | else if('isExpected' in definition) 40 | newDef.isExpected = isTrue; 41 | 42 | if(Object.keys(newDef).length) 43 | // only needed if there are any `ownProperties` 44 | forgivingStateDefinitions[key] = newDef; 45 | } 46 | for(let key in state){ 47 | if(ignore && ignore.has(key)) 48 | continue; 49 | if(!stateDefKeys.has(key)) 50 | forgivingStateDefinitions[key] = acceptingDefinition; 51 | } 52 | stateManagerMixin(this, forgivingStateDefinitions); 53 | } 54 | 55 | const GenericTask = (function(){ 56 | const Parent = Task; 57 | function GenericTask(step, state){ 58 | manipulateStateManagerValidation.call(this, state); 59 | Parent.call(this, step, state); 60 | } 61 | GenericTask.prototype = Object.create(Parent.prototype); 62 | return GenericTask; 63 | })(); 64 | 65 | const GenericStep = (function(){ 66 | 67 | const Parent = Step; 68 | function GenericStep(process, state){ 69 | var taskCtors = {}; 70 | for(let [key/*, taskState*/] of state.tasks) 71 | taskCtors[key] = GenericTask; 72 | manipulateStateManagerValidation.call(this, state, new Set('tasks')); 73 | Parent.call(this, process, state, taskCtors); 74 | } 75 | GenericStep.prototype = Object.create(Parent.prototype); 76 | return GenericStep; 77 | })(); 78 | 79 | const GenericProcess = (function(){ 80 | /** 81 | * The idea of GenericProcess is to be able to display the data of 82 | * an outdated or otherwise incompatible process in the interface, 83 | * without any means of changing the actual data. E.g. the Task don't 84 | * have any actions. 85 | */ 86 | const Parent = Process; 87 | function GenericProcess(resources, state) { 88 | // guessing by the data which step constructors are required by it. 89 | var stepCtors = null 90 | , FailStepCtor = 'failStep' in state ? GenericStep : null 91 | , FinallyStepCtor = 'finallyStep' in state ? GenericStep : null 92 | ; 93 | if('steps' in state) { 94 | stepCtors = []; 95 | for(let i=0,l=state.steps.length;i {processId}/{stepId}/{taskId} 8 | * 9 | * as a string: 10 | * we could go for indexes OR for ids, for a mixture of both, where it 11 | * makes sense AND for both in all cases which is redundant. 12 | * 13 | * indexes make sense for steps, because these are an ordered list 14 | * ids make sense for tasks, because they are "unordered" at least 15 | * semantically (there's an explicit order in the `tasks` list of the 16 | * step that we use) 17 | * 18 | * process: {processId} 19 | * steps: {index} 20 | * task: {id} OR {index}:{id} ? => redundant, but maybe helps to reduce errors, i.e. when refactoring? 21 | * callback: {name} 22 | * 23 | * having an id and an index for a task, we can create a fingerprint 24 | * for the step. though, we can also create a fingerprint by merely 25 | * using ids! just have to sort the ids before to create a canonical order. 26 | * So, tasks don't have indexes … 27 | */ 28 | function Path(processId, step, task) { 29 | Object.defineProperties(this, { 30 | processId: {value: processId, enumerable: true} 31 | , step: {value: step, enumerable: true} 32 | , task: {value: task, enumerable: true} 33 | }); 34 | } 35 | 36 | const _p = Path.prototype = Object.create(null); 37 | 38 | _p.toString = function() { 39 | return [...this].join('/'); 40 | }; 41 | 42 | _p[Symbol.iterator] = function* () { 43 | if(!this.processId) 44 | return; 45 | yield this.processId; 46 | 47 | if(!this.step) 48 | return; 49 | yield this.step; 50 | 51 | if(!this.task) 52 | return; 53 | yield this.task; 54 | }; 55 | 56 | Path.fromString = function(pathString) { 57 | return new Path(...pathString.split('/')); 58 | }; 59 | 60 | exports.Path = Path; 61 | -------------------------------------------------------------------------------- /containers/base/javascript/node/dispatcher/framework/Status.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | /* jshint esnext:true, node:true*/ 3 | 4 | 5 | /** 6 | * Making these items unique objects so it's very explicit what is meant. 7 | */ 8 | function StatusCode (status) { 9 | Object.defineProperty(this, 'status', { 10 | value: status 11 | , writable: false 12 | , enumerable: true 13 | }); 14 | } 15 | StatusCode.prototype.valueOf = function() { 16 | return this.status; 17 | }; 18 | 19 | StatusCode.prototype.toString = function() { 20 | return this.status; 21 | }; 22 | 23 | const PENDING = new StatusCode('PENDING') 24 | , OK = new StatusCode('OK') 25 | , FAILED = new StatusCode('FAILED') 26 | , LOG = new StatusCode('LOG') 27 | , statusCodes = new Map(Object.entries({PENDING, OK, FAILED, LOG})) 28 | // string2statusCode: statusCodes.get(string) => statusCode 29 | // statusCodes.get('FAILED') => FAILED 30 | , string2statusCode = string=>statusCodes.get(string) 31 | ; 32 | 33 | exports.PENDING = PENDING; 34 | exports.FAILED = FAILED; 35 | exports.OK = OK; 36 | exports.LOG = LOG; 37 | exports.statusCodes = statusCodes; 38 | exports.string2statusCode = string2statusCode; 39 | 40 | const Status = (function() { 41 | function Status( 42 | status/*status item*/ 43 | , details/*string(markdown)*/ 44 | , created/*Date: optional*/ 45 | , data/* losslessly JSON serializable data: optional*/) { 46 | // FIXME: vaidate types! 47 | 48 | //must be an existing status item 49 | if(!statusCodes.has(status.toString())) 50 | throw new Error('`status` "'+status+'" is unknown.'); 51 | this.status = status; 52 | 53 | if(typeof details !== 'string') 54 | throw new Error('`details` "'+details+'" is not a string "'+(typeof details)+'".'); 55 | this.details = details; // TODO: must be a string 56 | 57 | if(created) { 58 | // if present must be a valid date 59 | if(!(created instanceof Date)) 60 | throw new Error('`created` is not an instance of Date "'+created+'".'); 61 | if(isNaN(created.getDate())) 62 | throw new Error('`created` is an Invalid Date "'+created+'".'); 63 | this.created = created; 64 | } 65 | else 66 | this.created = new Date(); 67 | // for structured data in advanced situations 68 | this.data = data || null; 69 | } 70 | 71 | const _p = Status.prototype; 72 | 73 | _p.serialize = function() { 74 | return { 75 | status: this.status.toString() 76 | , details: this.details 77 | // for use with rethinkDB this can just stay a Date 78 | , created: this.created //.toISOString() 79 | , data: this.data 80 | }; 81 | }; 82 | 83 | /** 84 | * Just a factory function. 85 | */ 86 | Status.load = function(state) { 87 | var { 88 | status: statusString 89 | , details 90 | , created 91 | , data 92 | } = state; 93 | // stateString :must exist, check in here for a better error message 94 | if(!statusCodes.has(statusString)) 95 | throw new Error('state.status is not a statusCodes key: "'+statusString+'"'); 96 | 97 | // rethinkdb can store and return dates as it 98 | if(typeof created === 'string') 99 | created = new Date(created); 100 | return new Status( 101 | statusCodes.get(statusString) 102 | , details 103 | , created 104 | , data 105 | ); 106 | }; 107 | 108 | return Status; 109 | })(); 110 | 111 | exports.Status = Status; 112 | -------------------------------------------------------------------------------- /containers/base/javascript/node/util/AsyncQueue.js: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | "use strict"; 3 | // this is expected to run in nodejs 4 | /* jshint esnext:true, node:true*/ 5 | 6 | function AsyncQueue(onRunEmpty) { 7 | this._current = null; 8 | this._thread = []; 9 | // Used to e.g. remove the queue if it is created dynamically 10 | // and not needed anymore: 11 | // `onRunEmpty=()=>myDynamicQueues.delete(keyForNowEmptyQueue);` 12 | this._onRunEmpty = onRunEmpty; 13 | } 14 | 15 | const _p = AsyncQueue.prototype; 16 | 17 | _p._tick = function() { 18 | if(!this._thread.length || this._current) { 19 | if(!this._current && this._onRunEmpty) 20 | this._onRunEmpty(); 21 | return; 22 | } 23 | 24 | this._current = this._thread.pop(); 25 | this._current().then(() => { 26 | this._current = null; 27 | this._tick(); 28 | }); 29 | }; 30 | 31 | _p.schedule = function(job, ...args) { 32 | var resolve, reject 33 | // resolve, reject of the closure are 34 | , jobPromise = new Promise((res, rej) => { 35 | resolve = res; 36 | reject = rej; 37 | }) 38 | ; 39 | this._thread.unshift(() => { 40 | var result; 41 | try { 42 | result = job(...args); 43 | resolve(result); 44 | } catch(err) { 45 | reject(err); 46 | } 47 | 48 | return (result && typeof result.then === 'function') 49 | // run next after result; no matter if result succeeds or fails 50 | ? result.then(()=>null, ()=>null) 51 | // run next queue.thread item asap 52 | : Promise.resolve(null) 53 | ; 54 | }); 55 | this._tick(); 56 | return jobPromise; 57 | }; 58 | 59 | exports.AsyncQueue = AsyncQueue; 60 | -------------------------------------------------------------------------------- /containers/base/javascript/node/util/DispatcherProcessManagerClient.js: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | "use strict"; 3 | // this is expected to run in nodejs 4 | /* global require */ 5 | /* jshint esnext:true */ 6 | 7 | const { nodeCallback2Promise } = require('./nodeCallback2Promise') 8 | , { ProcessManagerClient:Parent } = require('./ProcessManagerClient') 9 | , { DispatcherProcessManagerClient: GrpcDispatcherProcessManagerClient 10 | } = require('protocolbuffers/messages_grpc_pb') 11 | , { DispatcherInitProcess } = require('protocolbuffers/messages_pb') 12 | ; 13 | 14 | /** 15 | * new DispatcherProcessManagerClient(logging, 'localhost', 1234) 16 | */ 17 | function DispatcherProcessManagerClient(...args) { 18 | var anySetup = { 19 | knownTypes: { DispatcherInitProcess } 20 | }; 21 | Parent.call(this, ...args, null, anySetup); 22 | // In a GRPC server I can add many services, but for the client, it 23 | // seems that I have to initialize separate clients. 24 | this._clientDispatcher = new GrpcDispatcherProcessManagerClient(...this._grpcClientArgs); 25 | } 26 | 27 | var _p = DispatcherProcessManagerClient.prototype = Object.create(Parent.prototype); 28 | 29 | _p.subscribeProcessList = function(processListQuery) { 30 | return this._getStreamAsGenerator(this._clientDispatcher 31 | , 'subscribeProcessList', processListQuery); 32 | }; 33 | 34 | _p.waitForReady = function() { 35 | return Promise.all([ 36 | Parent.prototype.waitForReady.call(this) 37 | , nodeCallback2Promise((callback)=> 38 | this._clientDispatcher.waitForReady(this.deadline, callback)) 39 | ]); 40 | }; 41 | 42 | exports.DispatcherProcessManagerClient = DispatcherProcessManagerClient; 43 | 44 | if (typeof require != 'undefined' && require.main==module) { 45 | throw new Error ('Does not implemented a CLI!'); 46 | } 47 | -------------------------------------------------------------------------------- /containers/base/javascript/node/util/GitHubAuthClient.js: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | "use strict"; 3 | // this is expected to run in nodejs 4 | /* global require */ 5 | /* jshint esnext:true */ 6 | 7 | const { nodeCallback2Promise } = require('./nodeCallback2Promise') 8 | , grpc = require('grpc') 9 | , { AuthServiceClient: GrpcAuthServiceClient } = require('protocolbuffers/messages_grpc_pb') 10 | , { Empty } = require('google-protobuf/google/protobuf/empty_pb.js') 11 | ; 12 | 13 | /** 14 | * new GitHubAuthClient(logging, 'localhost', 5678) 15 | * 16 | * 17 | * service AuthService { 18 | * rpc InitSession (google.protobuf.Empty) returns (AuthStatus) {}; 19 | * rpc Logout (SessionId) returns (google.protobuf.Empty) {}; 20 | * rpc Authorize (AuthorizeRequest) returns (AuthStatus) {}; 21 | * rpc CheckSession (SessionId) returns (AuthStatus) {}; 22 | * } 23 | */ 24 | function GitHubAuthClient(logging, host, port, credentials) { 25 | var address = [host, port].join(':'); 26 | this._log = logging; 27 | this._deadline = 30; 28 | this._log.info('GitHubAuthClient at:', address); 29 | this._client = new GrpcAuthServiceClient( 30 | address 31 | , credentials || grpc.credentials.createInsecure() 32 | , { 33 | 'grpc.max_send_message_length': 80 * 1024 * 1024 34 | , 'grpc.max_receive_message_length': 80 * 1024 * 1024 35 | } 36 | ); 37 | } 38 | 39 | var _p = GitHubAuthClient.prototype; 40 | _p.constructor = GitHubAuthClient; 41 | 42 | _p._raiseUnhandledError = function(err) { 43 | this._log.error(err); 44 | throw err; 45 | }; 46 | 47 | Object.defineProperty(_p, 'deadline', { 48 | get: function() { 49 | if(this._deadline === Infinity) 50 | return this._deadline; 51 | var deadline = new Date(); 52 | deadline.setSeconds(deadline.getSeconds() + this._deadline); 53 | return deadline; 54 | } 55 | }); 56 | 57 | 58 | _p.initSession = function(){ 59 | var message = new Empty(); 60 | return nodeCallback2Promise((callback)=> 61 | this._client.initSession(message, {deadline: this.deadline}, callback)) 62 | .then(null, error=>this._raiseUnhandledError(error)); 63 | }; 64 | 65 | _p.logout = function(sessionId) { 66 | return nodeCallback2Promise((callback)=> 67 | this._client.logout(sessionId, {deadline: this.deadline}, callback)) 68 | .then(null, error=>this._raiseUnhandledError(error)); 69 | }; 70 | 71 | _p.authorize = function(authorizeRequest) { 72 | return nodeCallback2Promise((callback)=> 73 | this._client.authorize(authorizeRequest, {deadline: this.deadline}, callback)) 74 | .then(null, error=>this._raiseUnhandledError(error)); 75 | }; 76 | 77 | _p.checkSession = function(sessionId) { 78 | return nodeCallback2Promise((callback)=> 79 | this._client.checkSession(sessionId, {deadline: this.deadline}, callback)) 80 | .then(null, error=>this._raiseUnhandledError(error)); 81 | }; 82 | 83 | _p.getRoles = function(authorizedRolesRequest){ 84 | return nodeCallback2Promise((callback)=> 85 | this._client.getRoles(authorizedRolesRequest, {deadline: this.deadline}, callback)) 86 | .then(null, error=>this._raiseUnhandledError(error)); 87 | }; 88 | 89 | _p.getOAuthToken = function(sessionId){ 90 | return nodeCallback2Promise((callback)=> 91 | this._client.getOAuthToken(sessionId, {deadline: this.deadline}, callback)) 92 | .then(null, error=>this._raiseUnhandledError(error)); 93 | }; 94 | 95 | 96 | _p.waitForReady = function() { 97 | return nodeCallback2Promise((callback)=> 98 | this._client.waitForReady(this.deadline, callback)) 99 | .then(null, error=>{throw new Error(this.constructor.name + '' + error);}); 100 | }; 101 | 102 | exports.GitHubAuthClient = GitHubAuthClient; 103 | 104 | if (typeof require != 'undefined' && require.main==module) { 105 | throw new Error ('Does not implemented a CLI!'); 106 | } 107 | -------------------------------------------------------------------------------- /containers/base/javascript/node/util/GitHubOperationsClient.js: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | "use strict"; 3 | // this is expected to run in nodejs 4 | /* global require */ 5 | /* jshint esnext:true */ 6 | 7 | const { nodeCallback2Promise } = require('./nodeCallback2Promise') 8 | , grpc = require('grpc') 9 | , { GitHubOperationsClient: grpcGitHubOperationsClient } = require('protocolbuffers/messages_grpc_pb') 10 | ; 11 | 12 | /** 13 | * new GitHubOperationsClient(logging, 'localhost', 5678) 14 | * 15 | * service GitHubOperations { 16 | * rpc Dispatch (PullRequest) returns (google.protobuf.Empty) {}; 17 | * } 18 | */ 19 | function GitHubOperationsClient(logging, host, port, credentials) { 20 | var address = [host, port].join(':'); 21 | this._log = logging; 22 | this._deadline = 30; 23 | this._log.info('GitHubOperationsClient at:', address); 24 | this._client = new grpcGitHubOperationsClient( 25 | address 26 | , credentials || grpc.credentials.createInsecure() 27 | , { 28 | 'grpc.max_send_message_length': 80 * 1024 * 1024 29 | , 'grpc.max_receive_message_length': 80 * 1024 * 1024 30 | } 31 | ); 32 | } 33 | 34 | var _p = GitHubOperationsClient.prototype; 35 | _p.constructor = GitHubOperationsClient; 36 | 37 | _p._raiseUnhandledError = function(err) { 38 | this._log.error(err); 39 | throw err; 40 | }; 41 | 42 | Object.defineProperty(_p, 'deadline', { 43 | get: function() { 44 | if(this._deadline === Infinity) 45 | return this._deadline; 46 | var deadline = new Date(); 47 | deadline.setSeconds(deadline.getSeconds() + this._deadline); 48 | return deadline; 49 | } 50 | }); 51 | 52 | _p.dispatchPullRequest = function(pullRequest) { 53 | return nodeCallback2Promise((callback)=> 54 | this._client.dispatchPullRequest(pullRequest, {deadline: this.deadline}, callback)) 55 | .then(null, error=>this._raiseUnhandledError(error)); 56 | }; 57 | 58 | // rpc FileIssue (Issue) returns (GitHubReport) {}; 59 | _p.fileIssue = function(issue) { 60 | return nodeCallback2Promise((callback)=> 61 | this._client.fileIssue(issue, {deadline: this.deadline}, callback)) 62 | .then(null, error=>this._raiseUnhandledError(error)); 63 | }; 64 | 65 | _p.waitForReady = function() { 66 | return nodeCallback2Promise((callback)=> 67 | this._client.waitForReady(this.deadline, callback)) 68 | .then(null, error=>{throw new Error(this.constructor.name + '' + error);}); 69 | }; 70 | 71 | exports.GitHubOperationsClient = GitHubOperationsClient; 72 | 73 | if (typeof require != 'undefined' && require.main==module) { 74 | throw new Error ('Does not implemented a CLI!'); 75 | } 76 | -------------------------------------------------------------------------------- /containers/base/javascript/node/util/IOOperations.js: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | "use strict"; 3 | // this is expected to run in nodejs 4 | /* global require */ 5 | /* jshint esnext:true */ 6 | 7 | 8 | const { initDB, initAmqp }= require('./getSetup') 9 | , { FamilyJob } = require('protocolbuffers/messages_pb') 10 | ; 11 | 12 | function IOOperations(logging, dbSetup, amqpSetup) { 13 | this._log = logging; 14 | this._dbSetup = dbSetup; 15 | this._amqpSetup = amqpSetup; 16 | this._initPromise = null; 17 | this._r = null; 18 | this._dbTables = this._dbSetup && this._dbSetup.tables || {}; 19 | this._amqp = null; 20 | } 21 | 22 | var _p = IOOperations.prototype; 23 | 24 | /** 25 | * same as pythons builtin `zip` function 26 | */ 27 | function zip(...arrays) { 28 | var result = []; 29 | for(let i=0,l=Math.min(...arrays.map(a=>a.length));i { 41 | promises.push(promise); 42 | names.push(name); 43 | } 44 | ; 45 | if(this._dbSetup) 46 | pushPromise('_r', initDB(this._log, this._dbSetup)); 47 | if(this._amqpSetup) 48 | pushPromise('_amqp', initAmqp(this._log, this._amqpSetup)); 49 | 50 | this._initPromise = Promise.all(promises) 51 | .then(resources=>{ 52 | let names_resources = zip(names, resources); 53 | for(let [name, resource] of names_resources) 54 | this[name] = resource; 55 | return resources; 56 | }); 57 | } 58 | return this._initPromise; 59 | }; 60 | 61 | Object.defineProperties(_p, { 62 | r: { 63 | get: function() { 64 | if(!this._r) 65 | throw new Error('Database resource "this._r" was not configured.'); 66 | return this._r; 67 | } 68 | } 69 | , amqp: { 70 | get: function() { 71 | if(!this._amqp) 72 | throw new Error('Messaging Queue resource "this._amqp" was not configured.'); 73 | return this._amqp; 74 | } 75 | } 76 | , hasAmqp: { 77 | get: function() { 78 | return !!this._amqp; 79 | } 80 | } 81 | 82 | }); 83 | 84 | _p.query = function(dbTable) { 85 | let r = this.r // raises if db was not configured; 86 | , realTableName = this._dbTables[dbTable] 87 | ; 88 | return r.table(realTableName); 89 | }; 90 | 91 | _p.insertDoc = function(dbTable, doc) { 92 | return this.query(dbTable).insert(doc).run() 93 | .error(function(err) { 94 | this._log.error('Creating a doc failed ', err); 95 | throw err; // re-raise 96 | }); 97 | }; 98 | 99 | _p.getLatesCollectionEntry = function(collection_id, family_name) { 100 | return this.query('collection') 101 | .getAll([collection_id, family_name], {index:'collection_family'}) 102 | .orderBy(this.r.desc('date')) 103 | .limit(1) 104 | // => should return just the first element 105 | // .nth(0) there's no description what it does if the element doesn't exist! 106 | // but it looks like we get an error 107 | .run() 108 | .then(entries=>entries[0]) // => entry or undefined 109 | ; 110 | }; 111 | 112 | _p.sendQueueMessage = function (queueName, message) { 113 | var options = { 114 | // TODO: do we need persistent here/always? 115 | persistent: true // same as deliveryMode: true or deliveryMode: 2 116 | } 117 | ; 118 | function sendMessage() { 119 | // jshint validthis:true 120 | // this._log.info('sendToQueue: ', queueName); 121 | return this.amqp.channel.sendToQueue(queueName, message, options); 122 | } 123 | return this.amqp.channel.assertQueue(queueName, {durable: true}) 124 | .then(sendMessage.bind(this)) 125 | ; 126 | }; 127 | 128 | _p.queueListen = function(channelName, consumer) { 129 | return this.amqp.channel.assertQueue(channelName) 130 | .then(reply=>this.amqp.channel.consume(reply.queue, consumer)); 131 | }; 132 | 133 | _p.ackQueueMessage = function(message) { 134 | this.amqp.channel.ack(message); 135 | }; 136 | 137 | exports.IOOperations = IOOperations; 138 | -------------------------------------------------------------------------------- /containers/base/javascript/node/util/InitWorkersClient.js: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | "use strict"; 3 | // this is expected to run in nodejs 4 | /* global require */ 5 | /* jshint esnext:true */ 6 | 7 | const { nodeCallback2Promise } = require('./nodeCallback2Promise') 8 | , grpc = require('grpc') 9 | , { InitWorkersClient: grpcInitWorkersClient } = require('protocolbuffers/messages_grpc_pb') 10 | , { pack, unpack } = require('./ProtobufAnyHandler') 11 | , { WorkerDescription, FamilyJob } = require('protocolbuffers/messages_pb') 12 | , { Empty } = require('google-protobuf/google/protobuf/empty_pb.js') 13 | ; 14 | 15 | /** 16 | * new InitWorkersClient(logging, 'localhost', 5678) 17 | * 18 | * service InitWorkers { 19 | * // the message type of the answer is worker implementation dependent. 20 | * rpc Init (WorkerDescription) returns (google.protobuf.Any) {}; 21 | * } 22 | */ 23 | function InitWorkersClient(logging, host, port, credentials) { 24 | var address = [host, port].join(':'); 25 | this._log = logging; 26 | this._deadline = 30; 27 | this._log.info('InitWorkersClient at:', address); 28 | this._client = new grpcInitWorkersClient( 29 | address 30 | , credentials || grpc.credentials.createInsecure() 31 | , { 32 | 'grpc.max_send_message_length': 80 * 1024 * 1024 33 | , 'grpc.max_receive_message_length': 80 * 1024 * 1024 34 | } 35 | ); 36 | } 37 | 38 | var _p = InitWorkersClient.prototype; 39 | _p.constructor = InitWorkersClient; 40 | 41 | _p._raiseUnhandledError = function(err) { 42 | this._log.error(err); 43 | throw err; 44 | }; 45 | 46 | Object.defineProperty(_p, 'deadline', { 47 | get: function() { 48 | if(this._deadline === Infinity) 49 | return this._deadline; 50 | var deadline = new Date(); 51 | deadline.setSeconds(deadline.getSeconds() + this._deadline); 52 | return deadline; 53 | } 54 | }); 55 | 56 | /** 57 | * rpc Init (WorkerDescription) returns (google.protobuf.Any) {}; 58 | */ 59 | _p.init = function(workerDescription) { 60 | return nodeCallback2Promise((callback)=> 61 | this._client.init(workerDescription, {deadline: this.deadline}, callback)) 62 | .then(null, error=>this._raiseUnhandledError(error)); 63 | }; 64 | 65 | 66 | /** 67 | * A higher level init with more detailed knowledge. 68 | */ 69 | _p.initialize = function(workerName, initMessage, processCommand/*optional*/) { 70 | // This setup is kind of annoying. 71 | var initMessageTypes = { 72 | 'fontbakery': 'StorageKey' 73 | , 'diffenator': 'StorageKey' 74 | , 'diffbrowsers': 'StorageKey' 75 | } 76 | , answerMessageTypes = { 77 | 'fontbakery': FamilyJob 78 | , 'diffenator': Empty 79 | , 'diffbrowsers': Empty 80 | } 81 | , workerDescription = new WorkerDescription() 82 | , any = pack(initMessage, initMessageTypes[workerName]) 83 | ; 84 | 85 | workerDescription.setWorkerName(workerName); 86 | workerDescription.setJob(any); 87 | if(processCommand) 88 | workerDescription.setProcessCommand(processCommand); 89 | 90 | return this.init(workerDescription) 91 | .then(anyAnswer=>{ 92 | var AnswerType = answerMessageTypes[workerName]; 93 | return unpack(anyAnswer, AnswerType); 94 | }); 95 | }; 96 | 97 | _p.waitForReady = function() { 98 | return nodeCallback2Promise((callback)=> 99 | this._client.waitForReady(this.deadline, callback)) 100 | .then(null, error=>{throw new Error(this.constructor.name + '' + error);}); 101 | }; 102 | 103 | exports.InitWorkersClient = InitWorkersClient; 104 | 105 | if (typeof require != 'undefined' && require.main==module) { 106 | throw new Error ('Does not implemented a CLI!'); 107 | } 108 | -------------------------------------------------------------------------------- /containers/base/javascript/node/util/Logging.js: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | "use strict"; 3 | // this is expected to run in nodejs 4 | /* jshint esnext:true */ 5 | 6 | 7 | function Logging(loglevel) { 8 | this._numericLoglevel = this._levels[loglevel] || 0; 9 | this._loglevel = loglevel; 10 | } 11 | 12 | var _p = Logging.prototype; 13 | 14 | Object.defineProperty(_p, 'loglevel', { 15 | get: function() { 16 | return this._loglevel; 17 | } 18 | }); 19 | 20 | 21 | _p.log = console.log; 22 | _p._levels = {}; 23 | ([ 24 | ['DEBUG', 10, console.info] 25 | , ['INFO', 20, console.info] 26 | , ['WARNING', 30, console.warn] 27 | , ['ERROR', 40, console.error] 28 | , ['CRITICAL', 50, console.error] 29 | ]).forEach(function(setup) { 30 | // method names: debug, info, warning, etc.. 31 | var loglevel = setup[0] 32 | , method = loglevel.toLowerCase() 33 | , numeric = setup[1] 34 | , log = setup[2] 35 | ; 36 | _p._levels[loglevel] = numeric; 37 | _p[method] = function() { 38 | if(numeric < this._numericLoglevel) 39 | return; 40 | var args = [loglevel], i, l; 41 | for(i=0,l=arguments.length;i 51 | this._client.poke(manifestSourceId, {deadline: this.deadline}, callback)) 52 | .then(null, error=>this._raiseUnhandledError(error)); 53 | }; 54 | 55 | // rpc Get (FamilyRequest) returns (FamilyData){} 56 | _p.get = function(familyRequest, deadline=this.deadline){ 57 | return nodeCallback2Promise((callback)=> 58 | this._client.get(familyRequest, {deadline: deadline}, callback)) 59 | .then(null, error=>this._raiseUnhandledError(error)); 60 | }; 61 | 62 | _p.getDelayed = function(familyRequest, deadline=this.deadline){ 63 | return nodeCallback2Promise((callback)=> 64 | this._client.getDelayed(familyRequest, {deadline: deadline}, callback)) 65 | .then(null, error=>this._raiseUnhandledError(error)); 66 | }; 67 | 68 | 69 | // rpc List (ManifestSourceId) returns (FamilyNamesList){} 70 | _p.list = function(manifestSourceId){ 71 | return nodeCallback2Promise((callback)=> 72 | this._client.list(manifestSourceId, {deadline: this.deadline}, callback)) 73 | .then(null, error=>this._raiseUnhandledError(error)); 74 | }; 75 | 76 | // rpc GetSourceDetails (FamilyRequest) returns (SourceDetails){} 77 | _p.getSourceDetails = function(familyRequest) { 78 | return nodeCallback2Promise((callback)=> 79 | this._client.getSourceDetails(familyRequest, {deadline: this.deadline}, callback)) 80 | .then(null, error=>this._raiseUnhandledError(error)); 81 | }; 82 | 83 | 84 | _p.waitForReady = function() { 85 | return nodeCallback2Promise((callback)=> 86 | this._client.waitForReady(this.deadline, callback)) 87 | .then(null, error=>{throw new Error(this.constructor.name + '' + error);}); 88 | }; 89 | 90 | exports.ManifestClient = ManifestClient; 91 | 92 | if (typeof require != 'undefined' && require.main==module) { 93 | throw new Error ('Does not implemented a CLI!'); 94 | } 95 | -------------------------------------------------------------------------------- /containers/base/javascript/node/util/ProtobufAnyHandler.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | /* jshint esnext:true, node:true*/ 3 | 4 | const { Any } = require('google-protobuf/google/protobuf/any_pb.js') 5 | , FONT_BAKERY_TYPES_NAMESPACE = 'fontbakery.dashboard' 6 | ; 7 | 8 | function ProtobufAnyHandler(logging, knownTypes, typesNamespace=FONT_BAKERY_TYPES_NAMESPACE) { 9 | this._log = logging; 10 | this._knownTypes = knownTypes || {}; 11 | this._typesNamespace = typesNamespace && typesNamespace.slice(-1) === '.' 12 | ? typesNamespace.slice(0, -1) 13 | : typesNamespace 14 | ; 15 | } 16 | 17 | const _p = ProtobufAnyHandler.prototype; 18 | 19 | _p.getTypeNameForMessage = function(message) { 20 | var name; 21 | for(name in this._knownTypes) 22 | if(message instanceof this._knownTypes[name]) 23 | return [this._typesNamespace, name].join('.'); 24 | this._log.debug('Unknown message type', message); 25 | throw new Error('Can\'t find type name for message'); 26 | }; 27 | 28 | _p.getTypeForTypeName = function(typeName) { 29 | var name = typeName.split('.').pop(); 30 | if(name in this._knownTypes) 31 | return this._knownTypes[name]; 32 | this._log.debug('Unknown type name ', typeName, 'known types are:' 33 | , Object.keys(this._knownTypes).join(', ')); 34 | throw new Error('Can\'t find type for type name,'); 35 | }; 36 | 37 | _p.pack = function(message) { 38 | var any = new Any() 39 | , typeName = this.getTypeNameForMessage(message) // e.g. 'fontbakery.dashboard.Files' 40 | ; 41 | any.pack(message.serializeBinary(), typeName); 42 | return any; 43 | }; 44 | 45 | _p.unpack = function(any) { 46 | var typeName = any.getTypeName() 47 | , Type = this.getTypeForTypeName(typeName) 48 | , message = any.unpack(Type.deserializeBinary, typeName) 49 | ; 50 | return message; 51 | }; 52 | 53 | 54 | ///// pack and unpack are just simple wrappers. ///// 55 | 56 | /** 57 | * The main case for pack is to default add FONT_BAKERY_TYPES_NAMESPACE 58 | * as the default `typesNamespace` argument. 59 | */ 60 | function pack(message, typeName, typesNamespace=FONT_BAKERY_TYPES_NAMESPACE) { 61 | var any = new Any() 62 | , typesNamespace_ = typesNamespace && typesNamespace.slice(-1) === '.' 63 | ? typesNamespace.slice(0, -1) 64 | : typesNamespace 65 | , fullTypeName = [typesNamespace_, typeName].join('.') 66 | ; 67 | any.pack(message.serializeBinary(), fullTypeName); 68 | return any; 69 | } 70 | 71 | /** 72 | * The main usage of unpack is to be better readable than 73 | * `Type.deserializeBinary(any.getValue_asU8());` 74 | * If typeName is set it returns null in case of a missmatch. 75 | */ 76 | function unpack(any, Type, typeName /*optional*/) { 77 | // In any.unpack: `if (this.getTypeName() == name) {` WHY??? 78 | // This still is just hope based, that Type.deserializeBinary 79 | // can load the value. Without that name detour, this is the 80 | // implementation: 81 | // `Type.deserializeBinary(any.getValue_asU8());` 82 | // So, if we expect a type, it suggests that we should know 83 | // the name of the type, but that's no real validation, since 84 | // the name comes along with the data and is thus just user input 85 | // as well. We got to trust that the data can be parsed with Type. 86 | // In conclusion, if we know what data to expect, we don't have 87 | // to use Any, it's just that the added TypeName is a nice 88 | // documentation and may be useful for debugging. 89 | var typeName_ = typeName || any.getTypeName() 90 | , message = any.unpack(Type.deserializeBinary, typeName_) 91 | ; 92 | return message; 93 | } 94 | 95 | exports.unpack = unpack; 96 | exports.pack = pack; 97 | 98 | exports.ProtobufAnyHandler = ProtobufAnyHandler; 99 | -------------------------------------------------------------------------------- /containers/base/javascript/node/util/ReportsClient.js: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | "use strict"; 3 | // this is expected to run in nodejs 4 | /* global require */ 5 | /* jshint esnext:true */ 6 | 7 | const { nodeCallback2Promise } = require('./nodeCallback2Promise') 8 | , grpc = require('grpc') 9 | , { ReportsClient: GrpcReportsClient } = require('protocolbuffers/messages_grpc_pb') 10 | //, { Empty } = require('google-protobuf/google/protobuf/empty_pb.js') 11 | ; 12 | 13 | /** 14 | * new ReportsClient(logging, 'localhost', 1234) 15 | */ 16 | function ReportsClient(logging, host, port, credentials) { 17 | var address = [host, port].join(':'); 18 | this._logging = logging; 19 | // in seconds, we use this to get an error when the channel is broken 20 | // 30 secconds is a lot time, still, under high load I guess this 21 | // can take some time. 5 seconds was sometimes not enough on my minikube 22 | // setup. 23 | // TODO: maybe we can have multiple retries with increasing deadlines 24 | // and still fail eventually. 25 | this._deadline = 30; 26 | this._logging.info('ReportsClient at:', address); 27 | this._client = new GrpcReportsClient( 28 | address 29 | , credentials || grpc.credentials.createInsecure() 30 | , { 31 | 'grpc.max_send_message_length': 80 * 1024 * 1024 32 | , 'grpc.max_receive_message_length': 80 * 1024 * 1024 33 | } 34 | ); 35 | } 36 | 37 | var _p = ReportsClient.prototype; 38 | 39 | _p._raiseUnhandledError = function(err) { 40 | this._log.error(err); 41 | throw err; 42 | }; 43 | 44 | Object.defineProperty(_p, 'deadline', { 45 | get: function() { 46 | if(this._deadline === Infinity) 47 | return this._deadline; 48 | var deadline = new Date(); 49 | deadline.setSeconds(deadline.getSeconds() + this._deadline); 50 | return deadline; 51 | } 52 | }); 53 | 54 | _p.file = function(report) { 55 | return nodeCallback2Promise((callback)=> 56 | this._client.file(report, {deadline: this.deadline}, callback)) 57 | .then(null, error=>this._raiseUnhandledError(error)); 58 | }; 59 | 60 | _p._getStream = function(method, message) { 61 | var METHOD = '[' + method.toUpperCase() + ']'; 62 | return new Promise((resolve, reject) => { 63 | // Instead of passing the method a request and callback, we pass it 64 | // a request and get a Readable stream object back. 65 | var call = this._client[method](message, {deadline: this.deadline}) 66 | , result = [] 67 | ; 68 | 69 | // The client can use the Readable’s 'data' event to read the server’s responses. 70 | // This event fires with each Feature message object until there are no more messages. 71 | // Errors in the 'data' callback will not cause the stream to be closed! 72 | call.on('data', report=>{ 73 | this._logging.debug(METHOD+' receiving a', [report.getType() 74 | , report.getTypeId(), report.getMethod()].join(':') 75 | , report.hasReported() && report.getReported().toDate() 76 | ); 77 | result.push(report); 78 | }); 79 | 80 | // The 'end' event indicates that the server has finished sending 81 | // and no errors occured. 82 | call.on('end', ()=>resolve(result)); 83 | 84 | // An error has occurred and the stream has been closed. 85 | call.on('error', error => { 86 | this._logging.error('reports ' + METHOD + ' on:error', error); 87 | reject(error); 88 | }); 89 | 90 | // Only one of 'error' or 'end' will be emitted. 91 | // Finally, the 'status' event fires when the server sends the status. 92 | call.on('status', status=>{ 93 | if (status.code !== grpc.status.OK) { 94 | this._logging.warning('reports ' + METHOD + ' on:status', status); 95 | // on:error should have rejected already OR on:end already 96 | // resolved! 97 | // reject(status); 98 | } 99 | }); 100 | 101 | }); 102 | }; 103 | 104 | _p.query = function(reportsQuery) { 105 | return this._getStream('query', reportsQuery); 106 | }; 107 | 108 | 109 | _p.get = function(reportIds) { 110 | return this._getStream('get', reportIds); 111 | }; 112 | 113 | /** 114 | * returns a ReportsQuery.Filter describing all possible filters 115 | * i.e. the value filters have all values set, all date filters are listed 116 | */ 117 | /* 118 | _p.describe = function() { 119 | return nodeCallback2Promise((callback)=> 120 | this._client.query(new Empty, {deadline: this.deadline}, callback)) 121 | .then(null, error=>this._raiseUnhandledError(error)); 122 | } 123 | */ 124 | 125 | _p.waitForReady = function() { 126 | return nodeCallback2Promise((callback)=> 127 | this._client.waitForReady(this.deadline, callback)); 128 | }; 129 | 130 | exports.ReportsClient = ReportsClient; 131 | 132 | if (typeof require != 'undefined' && require.main==module) { 133 | throw new Error ('Does not implemented a CLI!'); 134 | } 135 | -------------------------------------------------------------------------------- /containers/base/javascript/node/util/nodeCallback2Promise.js: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | "use strict"; 3 | // this is expected to run in nodejs 4 | /* jshint esnext:true */ 5 | 6 | 7 | // Turn node callback style into a promise style 8 | function nodeCallback2Promise(func, ...args) { 9 | return new Promise(function(resolve, reject) { 10 | function callback(err, result) { 11 | if(err) reject(err); 12 | else resolve(result); 13 | } 14 | // callback is the last argument 15 | func(...args, callback); 16 | }); 17 | } 18 | 19 | exports.nodeCallback2Promise = nodeCallback2Promise; 20 | -------------------------------------------------------------------------------- /containers/base/javascript/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "fontbakery-dashboard", 3 | "version": "0.0.1", 4 | "description": "Javascript modules of the Font Bakery dashboard.", 5 | "private": true, 6 | "main": "index.js", 7 | "scripts": { 8 | "test": "echo \"Error: no test specified\" && exit 1", 9 | "start": "node node/api.js" 10 | }, 11 | "keywords": [ 12 | "font", 13 | "testing", 14 | "fontbakery" 15 | ], 16 | "author": "Lasse Fister ", 17 | "license": "Apache-2.0", 18 | "dependencies": { 19 | "amqplib": "^0.5.6", 20 | "body-parser": "^1.19.0", 21 | "cookie-parser": "^1.4.5", 22 | "csv-parse": "^4.10.1", 23 | "express": "^4.17.1", 24 | "google-protobuf": "^3.12.2", 25 | "grpc": "^1.24.3", 26 | "marked": "^0.7.0", 27 | "mime": "^2.4.6", 28 | "nodegit": "^0.26.2", 29 | "rethinkdbdash": "^2.3.31", 30 | "socket.io": "^2.3.0", 31 | "tmp": "0.0.33", 32 | "uid-safe": "^2.1.5", 33 | "yauzl": "^2.10.0", 34 | "yazl": "^2.5.1" 35 | }, 36 | "devDependencies": { 37 | "grpc-tools": "^1.9.0" 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /containers/base/protocolbuffers/shared.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package fontbakery.dashboard; 4 | 5 | message File { 6 | string name = 1; 7 | bytes data = 2; 8 | }; 9 | 10 | message Files { 11 | repeated File files = 1; 12 | }; 13 | -------------------------------------------------------------------------------- /containers/base/python/Dockerfile: -------------------------------------------------------------------------------- 1 | # eoan == 19.08 NOT LTS End of Life: July, 2020 2 | FROM ubuntu:eoan 3 | 4 | RUN touch /tmp/cache_spoof_7 5 | 6 | 7 | # START TZCONFIG FIX suggested in: 8 | # https://stackoverflow.com/a/47909037/1315369 9 | # The tzdata package was expecting interaction to configure it 10 | 11 | ## for apt to be noninteractive 12 | ENV DEBIAN_FRONTEND noninteractive 13 | ENV DEBCONF_NONINTERACTIVE_SEEN true 14 | 15 | ## preset tzdata, update package index, upgrade packages and install needed software 16 | RUN echo "tzdata tzdata/Areas select Europe" > /tmp/preseed.txt; \ 17 | echo "tzdata tzdata/Zones/Europe select Berlin" >> /tmp/preseed.txt; \ 18 | debconf-set-selections /tmp/preseed.txt && \ 19 | rm -f /etc/timezone && \ 20 | rm -f /etc/localtime && \ 21 | apt-get update && \ 22 | apt-get install -y tzdata 23 | 24 | ## cleanup of files from setup 25 | RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 26 | # END TZCONFIG FIX 27 | 28 | # universe in bionic has a Package: fontforge (1:20170731~dfsg-1) 29 | # while the ppa currently has no newer version and no bionic version yet 30 | # TODO: re-check and insrt the following line before the "&& apt-get update \" line 31 | # && add-apt-repository ppa:fontforge/fontforge \ 32 | 33 | RUN apt-get update && apt-get upgrade -y \ 34 | && apt-get install -y software-properties-common \ 35 | && apt-get update \ 36 | && apt-get install -y curl libicu-dev \ 37 | fontforge-nox python-fontforge git build-essential \ 38 | libssl-dev libffi-dev python-dev libxmlsec1-dev \ 39 | libxml2 libxml2-dev tzdata python3-minimal; 40 | 41 | # diffenator dependencies, to install pycairo 42 | RUN apt-get install -y python3-dev libcairo2 libcairo2-dev libharfbuzz-bin 43 | 44 | 45 | # RUN curl -o /tmp/get-pip.py https://bootstrap.pypa.io/get-pip.py; python /tmp/get-pip.py; 46 | RUN curl -o /tmp/get-pip.py https://bootstrap.pypa.io/get-pip.py; python3 /tmp/get-pip.py; 47 | 48 | run pip3 install --upgrade pip 49 | 50 | # FIXME! 51 | # Also, fontbakery should come with all requirements in it's setup.py 52 | # how does fontTools or defcon handle this? 53 | # ADD requirements.txt / 54 | # RUN pip3 install --no-cache-dir -r requirements.txt 55 | 56 | # FIXME: need all external dependencies 57 | # ADD prebuilt/ot-sanitise /usr/local/bin/ 58 | 59 | # This is taking forever to compile, so it slows down reiterations 60 | # it's a dependency of fontbakery. Adding it early, so it is cached 61 | # by the docker build 62 | # This is bad: fontbakery requirements.txt needs a version and setup.py doesn't, we 63 | # install from both ... 64 | RUN pip3 install lxml; # ==3.5.0 65 | 66 | ADD requirements.txt /var/python/requirements.txt 67 | RUN pip3 install -r /var/python/requirements.txt 68 | 69 | RUN touch /tmp/cache_spoof_4 70 | 71 | # FIXME: doesn't install all dependencies! 72 | # RUN pip3 install git+https://github.com/graphicore/fontbakery@dashboard_related 73 | 74 | #RUN git clone --depth 1 -b p3_fixes git://github.com/graphicore/fontbakery /var/fontbakery;\ 75 | #RUN git clone --depth 1 -b main git://github.com/googlefonts/fontbakery /var/fontbakery;\ 76 | # pip3 install /var/fontbakery; 77 | 78 | # FIXME: probably all `pip3 install dependencies should be in requirements.txt 79 | 80 | # See: https://github.com/fonttools/ttfautohint-py/issues/5 81 | # RUN apt-get install -y flex autotools-dev libtool bison cmake 82 | # RUN pip install scikit-build 83 | # RUN pip install git+https://github.com/fonttools/ttfautohint-py.git@v0.4.3#egg=ttfautohint-py 84 | 85 | RUN pip3 install fontbakery; 86 | 87 | 88 | # There's a temporary error in Pillow after 7.0.0: 89 | # https://github.com/python-pillow/Pillow/issues/4518#issuecomment-616377174 90 | # At version 7.1.1: "Downgrade to Pillow 7.0.0 until a newer Pillow is released" 91 | 92 | RUN pip3 -v install Pillow==7.0.0; 93 | 94 | # tools used for live debugging and profiling 95 | # RUN pip3 install memory-profiler 96 | # RUN apt-get install -y vim tmux htop 97 | # RUN touch /tmp/cache_spoof_1 98 | # RUN pip3 install remote-pdb 99 | # RUN git clone --depth 1 -b fb_dashboard_bughunt_issue100_OOM https://github.com/graphicore/fontdiffenator.git /var/fontdiffenator;\ 100 | # pip3 install -e /var/fontdiffenator; 101 | RUN pip3 install fontdiffenator; 102 | 103 | RUN pip3 install gfdiffbrowsers 104 | 105 | RUN touch /tmp/cache_spoof_1 106 | 107 | # in update_protobufs.sh the command `python -m grpc_tools.protoc` 108 | # doesn't create python 3 ready relative imports 109 | # see: https://github.com/protocolbuffers/protobuf/issues/1491 110 | # setting PYTHONPATH ths way helps. 111 | ENV PYTHONPATH="/var/python/protocolbuffers/:${PYTHONPATH}" 112 | 113 | # Solves isses raising errors like: 114 | # UnicodeEncodeError: 'ascii' codec can't encode characters in position 0-8: ordinal not in range(128) 115 | # See e.g.: https://github.com/docker-library/python/issues/13 116 | # also an option: ENV PYTHONIOENCODING=UTF-8 117 | ENV LANG C.UTF-8 118 | 119 | 120 | ADD . /var/python/ 121 | -------------------------------------------------------------------------------- /containers/base/python/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/__init__.py -------------------------------------------------------------------------------- /containers/base/python/debug_run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import logging 4 | FORMAT = '%(asctime)s:%(name)s:%(levelname)s:%(message)s' 5 | logging.basicConfig(format=FORMAT) 6 | 7 | import sys 8 | print('python version:', sys.version) 9 | 10 | from worker.diffenator import main 11 | main() 12 | -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/after/Vollkorn-Black.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/after/Vollkorn-Black.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/after/Vollkorn-BlackItalic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/after/Vollkorn-BlackItalic.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/after/Vollkorn-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/after/Vollkorn-Bold.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/after/Vollkorn-BoldItalic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/after/Vollkorn-BoldItalic.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/after/Vollkorn-ExtraBold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/after/Vollkorn-ExtraBold.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/after/Vollkorn-ExtraBoldItalic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/after/Vollkorn-ExtraBoldItalic.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/after/Vollkorn-Italic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/after/Vollkorn-Italic.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/after/Vollkorn-Medium.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/after/Vollkorn-Medium.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/after/Vollkorn-MediumItalic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/after/Vollkorn-MediumItalic.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/after/Vollkorn-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/after/Vollkorn-Regular.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/after/Vollkorn-SemiBold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/after/Vollkorn-SemiBold.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/after/Vollkorn-SemiBoldItalic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/after/Vollkorn-SemiBoldItalic.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/after/VollkornSC-Black.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/after/VollkornSC-Black.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/after/VollkornSC-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/after/VollkornSC-Bold.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/after/VollkornSC-ExtraBold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/after/VollkornSC-ExtraBold.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/after/VollkornSC-Medium.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/after/VollkornSC-Medium.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/after/VollkornSC-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/after/VollkornSC-Regular.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/after/VollkornSC-SemiBold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/after/VollkornSC-SemiBold.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/after/vf/Vollkorn-Italic-VF.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/after/vf/Vollkorn-Italic-VF.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/after/vf/Vollkorn-Roman-VF.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/after/vf/Vollkorn-Roman-VF.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/before/OFL.txt: -------------------------------------------------------------------------------- 1 | Copyright 2017 The Vollkorn Project Authors (https://github.com/FAlthausen/Vollkorn-Typeface) 2 | 3 | This Font Software is licensed under the SIL Open Font License, Version 1.1. 4 | This license is copied below, and is also available with a FAQ at: 5 | http://scripts.sil.org/OFL 6 | 7 | 8 | ----------------------------------------------------------- 9 | SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 10 | ----------------------------------------------------------- 11 | 12 | PREAMBLE 13 | The goals of the Open Font License (OFL) are to stimulate worldwide 14 | development of collaborative font projects, to support the font creation 15 | efforts of academic and linguistic communities, and to provide a free and 16 | open framework in which fonts may be shared and improved in partnership 17 | with others. 18 | 19 | The OFL allows the licensed fonts to be used, studied, modified and 20 | redistributed freely as long as they are not sold by themselves. The 21 | fonts, including any derivative works, can be bundled, embedded, 22 | redistributed and/or sold with any software provided that any reserved 23 | names are not used by derivative works. The fonts and derivatives, 24 | however, cannot be released under any other type of license. The 25 | requirement for fonts to remain under this license does not apply 26 | to any document created using the fonts or their derivatives. 27 | 28 | DEFINITIONS 29 | "Font Software" refers to the set of files released by the Copyright 30 | Holder(s) under this license and clearly marked as such. This may 31 | include source files, build scripts and documentation. 32 | 33 | "Reserved Font Name" refers to any names specified as such after the 34 | copyright statement(s). 35 | 36 | "Original Version" refers to the collection of Font Software components as 37 | distributed by the Copyright Holder(s). 38 | 39 | "Modified Version" refers to any derivative made by adding to, deleting, 40 | or substituting -- in part or in whole -- any of the components of the 41 | Original Version, by changing formats or by porting the Font Software to a 42 | new environment. 43 | 44 | "Author" refers to any designer, engineer, programmer, technical 45 | writer or other person who contributed to the Font Software. 46 | 47 | PERMISSION & CONDITIONS 48 | Permission is hereby granted, free of charge, to any person obtaining 49 | a copy of the Font Software, to use, study, copy, merge, embed, modify, 50 | redistribute, and sell modified and unmodified copies of the Font 51 | Software, subject to the following conditions: 52 | 53 | 1) Neither the Font Software nor any of its individual components, 54 | in Original or Modified Versions, may be sold by itself. 55 | 56 | 2) Original or Modified Versions of the Font Software may be bundled, 57 | redistributed and/or sold with any software, provided that each copy 58 | contains the above copyright notice and this license. These can be 59 | included either as stand-alone text files, human-readable headers or 60 | in the appropriate machine-readable metadata fields within text or 61 | binary files as long as those fields can be easily viewed by the user. 62 | 63 | 3) No Modified Version of the Font Software may use the Reserved Font 64 | Name(s) unless explicit written permission is granted by the corresponding 65 | Copyright Holder. This restriction only applies to the primary font name as 66 | presented to the users. 67 | 68 | 4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font 69 | Software shall not be used to promote, endorse or advertise any 70 | Modified Version, except to acknowledge the contribution(s) of the 71 | Copyright Holder(s) and the Author(s) or with their explicit written 72 | permission. 73 | 74 | 5) The Font Software, modified or unmodified, in part or in whole, 75 | must be distributed entirely under this license, and must not be 76 | distributed under any other license. The requirement for fonts to 77 | remain under this license does not apply to any document created 78 | using the Font Software. 79 | 80 | TERMINATION 81 | This license becomes null and void if any of the above conditions are 82 | not met. 83 | 84 | DISCLAIMER 85 | THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 86 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF 87 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT 88 | OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE 89 | COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 90 | INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL 91 | DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 92 | FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM 93 | OTHER DEALINGS IN THE FONT SOFTWARE. 94 | -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/before/Vollkorn-Black.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/before/Vollkorn-Black.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/before/Vollkorn-BlackItalic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/before/Vollkorn-BlackItalic.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/before/Vollkorn-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/before/Vollkorn-Bold.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/before/Vollkorn-BoldItalic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/before/Vollkorn-BoldItalic.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/before/Vollkorn-Italic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/before/Vollkorn-Italic.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/before/Vollkorn-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/before/Vollkorn-Regular.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/before/Vollkorn-SemiBold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/before/Vollkorn-SemiBold.ttf -------------------------------------------------------------------------------- /containers/base/python/debug_vollkorn/before/Vollkorn-SemiBoldItalic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/debug_vollkorn/before/Vollkorn-SemiBoldItalic.ttf -------------------------------------------------------------------------------- /containers/base/python/protocolbuffers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/base/python/protocolbuffers/__init__.py -------------------------------------------------------------------------------- /containers/base/python/protocolbuffers/fonts_public_pb2_grpc.py: -------------------------------------------------------------------------------- 1 | # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! 2 | import grpc 3 | 4 | -------------------------------------------------------------------------------- /containers/base/python/protocolbuffers/shared_pb2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by the protocol buffer compiler. DO NOT EDIT! 3 | # source: shared.proto 4 | 5 | from google.protobuf import descriptor as _descriptor 6 | from google.protobuf import message as _message 7 | from google.protobuf import reflection as _reflection 8 | from google.protobuf import symbol_database as _symbol_database 9 | # @@protoc_insertion_point(imports) 10 | 11 | _sym_db = _symbol_database.Default() 12 | 13 | 14 | 15 | 16 | DESCRIPTOR = _descriptor.FileDescriptor( 17 | name='shared.proto', 18 | package='fontbakery.dashboard', 19 | syntax='proto3', 20 | serialized_options=None, 21 | serialized_pb=b'\n\x0cshared.proto\x12\x14\x66ontbakery.dashboard\"\"\n\x04\x46ile\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\"2\n\x05\x46iles\x12)\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x1a.fontbakery.dashboard.Fileb\x06proto3' 22 | ) 23 | 24 | 25 | 26 | 27 | _FILE = _descriptor.Descriptor( 28 | name='File', 29 | full_name='fontbakery.dashboard.File', 30 | filename=None, 31 | file=DESCRIPTOR, 32 | containing_type=None, 33 | fields=[ 34 | _descriptor.FieldDescriptor( 35 | name='name', full_name='fontbakery.dashboard.File.name', index=0, 36 | number=1, type=9, cpp_type=9, label=1, 37 | has_default_value=False, default_value=b"".decode('utf-8'), 38 | message_type=None, enum_type=None, containing_type=None, 39 | is_extension=False, extension_scope=None, 40 | serialized_options=None, file=DESCRIPTOR), 41 | _descriptor.FieldDescriptor( 42 | name='data', full_name='fontbakery.dashboard.File.data', index=1, 43 | number=2, type=12, cpp_type=9, label=1, 44 | has_default_value=False, default_value=b"", 45 | message_type=None, enum_type=None, containing_type=None, 46 | is_extension=False, extension_scope=None, 47 | serialized_options=None, file=DESCRIPTOR), 48 | ], 49 | extensions=[ 50 | ], 51 | nested_types=[], 52 | enum_types=[ 53 | ], 54 | serialized_options=None, 55 | is_extendable=False, 56 | syntax='proto3', 57 | extension_ranges=[], 58 | oneofs=[ 59 | ], 60 | serialized_start=38, 61 | serialized_end=72, 62 | ) 63 | 64 | 65 | _FILES = _descriptor.Descriptor( 66 | name='Files', 67 | full_name='fontbakery.dashboard.Files', 68 | filename=None, 69 | file=DESCRIPTOR, 70 | containing_type=None, 71 | fields=[ 72 | _descriptor.FieldDescriptor( 73 | name='files', full_name='fontbakery.dashboard.Files.files', index=0, 74 | number=1, type=11, cpp_type=10, label=3, 75 | has_default_value=False, default_value=[], 76 | message_type=None, enum_type=None, containing_type=None, 77 | is_extension=False, extension_scope=None, 78 | serialized_options=None, file=DESCRIPTOR), 79 | ], 80 | extensions=[ 81 | ], 82 | nested_types=[], 83 | enum_types=[ 84 | ], 85 | serialized_options=None, 86 | is_extendable=False, 87 | syntax='proto3', 88 | extension_ranges=[], 89 | oneofs=[ 90 | ], 91 | serialized_start=74, 92 | serialized_end=124, 93 | ) 94 | 95 | _FILES.fields_by_name['files'].message_type = _FILE 96 | DESCRIPTOR.message_types_by_name['File'] = _FILE 97 | DESCRIPTOR.message_types_by_name['Files'] = _FILES 98 | _sym_db.RegisterFileDescriptor(DESCRIPTOR) 99 | 100 | File = _reflection.GeneratedProtocolMessageType('File', (_message.Message,), { 101 | 'DESCRIPTOR' : _FILE, 102 | '__module__' : 'shared_pb2' 103 | # @@protoc_insertion_point(class_scope:fontbakery.dashboard.File) 104 | }) 105 | _sym_db.RegisterMessage(File) 106 | 107 | Files = _reflection.GeneratedProtocolMessageType('Files', (_message.Message,), { 108 | 'DESCRIPTOR' : _FILES, 109 | '__module__' : 'shared_pb2' 110 | # @@protoc_insertion_point(class_scope:fontbakery.dashboard.Files) 111 | }) 112 | _sym_db.RegisterMessage(Files) 113 | 114 | 115 | # @@protoc_insertion_point(module_scope) 116 | -------------------------------------------------------------------------------- /containers/base/python/protocolbuffers/shared_pb2_grpc.py: -------------------------------------------------------------------------------- 1 | # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! 2 | import grpc 3 | 4 | -------------------------------------------------------------------------------- /containers/base/python/requirements.txt: -------------------------------------------------------------------------------- 1 | pika 2 | rethinkdb 3 | pytz 4 | requests 5 | protobuf>=3.4.0 6 | grpcio 7 | -------------------------------------------------------------------------------- /containers/base/python/worker/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /containers/base/python/worker/diffbrowsers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import print_function, division, unicode_literals 3 | 4 | import os 5 | 6 | from .worker_base import WorkerError 7 | from .diff_tools_shared import ( 8 | DiffWorkerBase 9 | , on_each_matching_or_new_font 10 | ) 11 | 12 | from collections import namedtuple 13 | 14 | from fontTools.ttLib import TTFont 15 | import json 16 | 17 | import diffbrowsers 18 | from diffbrowsers.diffbrowsers import DiffBrowsers 19 | from diffbrowsers.browsers import test_browsers 20 | 21 | from diffenator.font import DFont 22 | 23 | 24 | def _repeated_mkdir(directory_path): 25 | try: 26 | os.mkdir(directory_path) 27 | except FileExistsError: 28 | pass 29 | 30 | ################# 31 | # START taken from gftools-qa (and now modified!) 32 | # https://github.com/googlefonts/gftools/blob/main/bin/gftools-qa.py 33 | ################# 34 | 35 | def run_plot_glyphs(font_path, out): 36 | font_filename = os.path.basename(font_path)[:-4] 37 | dfont = DFont(font_path) 38 | if dfont.is_variable: 39 | for coords in dfont.instances_coordinates: 40 | dfont.set_variations(coords) 41 | img_out = os.path.join(out, "%s_%s.png" % ( 42 | font_filename, _instance_coords_to_filename(coords) 43 | )) 44 | dfont.glyphs.to_png(img_out, limit=100000) 45 | else: 46 | img_out = os.path.join(out, font_filename + ".png") 47 | dfont.glyphs.to_png(dst=img_out) 48 | 49 | def run_browser_previews(font_path, out, auth, gfr_url): 50 | browsers_to_test = test_browsers["vf_browsers"] 51 | font_name = os.path.basename(font_path)[:-4] 52 | diff_browsers = DiffBrowsers( 53 | auth=auth, 54 | gfr_instance_url=gfr_url, 55 | dst_dir=os.path.join(out, font_name), 56 | browsers=browsers_to_test, 57 | gfr_is_local=False) 58 | diff_browsers.new_session([font_path], [font_path]) 59 | diff_browsers.diff_view("waterfall") 60 | diff_browsers.diff_view("glyphs_all", pt=15) 61 | 62 | def run_browser_diffs(font_before, font_after, out, auth, gfr_url): 63 | browsers_to_test = test_browsers["vf_browsers"] 64 | diff_browsers = DiffBrowsers( 65 | auth=auth, 66 | gfr_instance_url=gfr_url, 67 | dst_dir=out, 68 | browsers=browsers_to_test, 69 | gfr_is_local=False) 70 | diff_browsers.new_session([font_before], 71 | [font_after]) 72 | diff_browsers.diff_view("waterfall") 73 | has_vfs = any([ 74 | 'fvar' in TTFont(font_before).keys(), 75 | 'fvar' in TTFont(font_after).keys() 76 | ]) 77 | info = os.path.join(out, "info.json") 78 | json.dump(diff_browsers.stats, open(info, "w")) 79 | if has_vfs: 80 | for i in range(15, 17): 81 | diff_browsers.diff_view("glyphs_all", pt=i) 82 | 83 | @on_each_matching_or_new_font 84 | def run_renderers(logger, font_before, font_after, out, auth, gfr_url): 85 | # CAUTION: font_before MAY be None if font_after is new and not an 86 | # update. Happens e.g. if a family is updated with new styles! 87 | if font_before is not None: 88 | # this is a "matching" font, an update (font_after) of an existing 89 | # font (font_before) 90 | logger.debug('run_browser_diffs with fonts before: %s after: %s' 91 | , font_before, font_after) 92 | browser_diffs_out = os.path.join(out, "Browser_Diffs") 93 | _repeated_mkdir(out) 94 | _repeated_mkdir(browser_diffs_out) 95 | run_browser_diffs(font_before, font_after, browser_diffs_out, auth, gfr_url) 96 | 97 | else: 98 | # this is a new font, hence there's no font_before 99 | # but we can render previews for the font 100 | logger.debug('run_plot_glyphs with fonts: %s', font_after) 101 | plot_glyphs_out = os.path.join(out, "Plot_Glyphs") 102 | _repeated_mkdir(out) 103 | _repeated_mkdir(plot_glyphs_out) 104 | run_plot_glyphs(font_after, plot_glyphs_out) 105 | 106 | logger.debug('run_browser_previews with fonts: %s', font_after) 107 | browser_previews_out = os.path.join(out, "Browser_Previews") 108 | _repeated_mkdir(browser_previews_out) 109 | run_browser_previews(font_after, browser_previews_out, auth, gfr_url) 110 | 111 | ################# 112 | # /END taken from gftools-qa 113 | ################# 114 | 115 | Setup = namedtuple('Setup', ['gfr_url', 'bstack_credentials']) 116 | 117 | def getSetup(): 118 | gfr_url = os.environ.get("GF_REGRESSIONS_URL", "http://35.238.63.0/") 119 | 120 | bstack_username = os.environ.get("BROWSERSTACK_USERNAME", None) 121 | bstack_access_key = os.environ.get("BROWSERSTACK_ACCESS_KEY", None) 122 | if bstack_username is None or bstack_access_key is None: 123 | raise WorkerError('Browserstack authentication ' 124 | 'information is missing, please set the ' 125 | 'BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY ' 126 | 'environment variables.') 127 | bstack_credentials = (bstack_username, bstack_access_key) 128 | 129 | return Setup(gfr_url, bstack_credentials) 130 | 131 | # Running this rather really early than when the worker is initialized 132 | # the first time. That way we get feedback when configuration is missing 133 | # directly when the module is loaded. I.e. in this case directly on process 134 | # start. 135 | SETUP = getSetup(); 136 | 137 | class DiffbrowsersWorker(DiffWorkerBase): 138 | def __init__(self, logging, job, cache, persistence, queue, tmp_directory): 139 | self._workername = 'diffbrowsers' 140 | super().__init__(logging, job, cache, persistence, queue, tmp_directory) 141 | self._answer.preparation_logs.append( 142 | 'Diffbrowsers version {}'.format(diffbrowsers.__version__)) 143 | 144 | def run(self): 145 | self._set_answer_timestamp('started') 146 | fonts = self._prepare(self._cache.get(self._job.cache_key).files, ['before', 'after']) 147 | # all_fonts = reduce(lambda a,b: a+b, fonts.values(),[]) 148 | all_files = [os.path.join(dp, f) for dp, dn, fn \ 149 | in os.walk(self._tmp_directory) for f in fn] 150 | self._log.debug('Files in Tempdir {}: {}'.format( 151 | self._tmp_directory, all_files)) 152 | 153 | gfr_url = SETUP.gfr_url 154 | bstack_credentials = SETUP.bstack_credentials 155 | 156 | self._log.info('entering run_renderers …') 157 | # FIXME: should we collect stdout/stderr here??? 158 | run_renderers(self._log, fonts['before'], fonts['after'] 159 | , self._out_dir, bstack_credentials, gfr_url) 160 | self._log.info('DONE! docid: %s', self._job.docid) 161 | -------------------------------------------------------------------------------- /containers/base/python/worker/diffenator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import print_function, division, unicode_literals 3 | 4 | import os 5 | 6 | from .diff_tools_shared import ( 7 | DiffWorkerBase 8 | , on_each_matching_font 9 | ) 10 | 11 | import diffenator 12 | from diffenator.diff import DiffFonts 13 | from diffenator.font import DFont 14 | 15 | 16 | ################# 17 | # START taken from gftools-qa 18 | # https://github.com/googlefonts/gftools/blob/main/bin/gftools-qa.py 19 | ################# 20 | DIFFENATOR_THRESHOLDS = { 21 | "weak": dict( 22 | glyphs_thresh=0.02, 23 | marks_thresh=20, 24 | mkmks_thresh=20, 25 | kerns_thresh=30, 26 | render_diffs=True, 27 | ), 28 | "normal": dict( 29 | glyphs_thresh=0.01, 30 | marks_thresh=10, 31 | mkmks_thresh=10, 32 | kerns_thresh=15, 33 | render_diffs=True, 34 | ), 35 | "strict": dict( 36 | glyphs_thresh=0.00, 37 | marks_thresh=0, 38 | mkmks_thresh=0, 39 | kerns_thresh=1, 40 | render_diffs=True, 41 | ) 42 | } 43 | 44 | @on_each_matching_font 45 | def run_diffenator(logger, font_before, font_after, out, thresholds=DIFFENATOR_THRESHOLDS['normal']): 46 | logger.debug('run_diffenator with fonts before: %s after: %s' 47 | , font_before, font_after) 48 | 49 | font_before = DFont(font_before) 50 | font_after = DFont(font_after) 51 | 52 | 53 | if font_after.is_variable and not font_before.is_variable: 54 | font_after.set_variations_from_static(font_before) 55 | 56 | elif not font_after.is_variable and font_before.is_variable: 57 | font_before.set_variations_from_static(font_after) 58 | 59 | elif font_after.is_variable and font_before.is_variable: 60 | # TODO get wdth and slnt axis vals 61 | variations = {"wght": font_before.ttfont["OS/2"].usWeightClass} 62 | font_after.set_variations(variations) 63 | font_before.set_variations(variations) 64 | 65 | diff = DiffFonts(font_before, font_after, settings=thresholds) 66 | diff.to_gifs(dst=out) 67 | diff.to_txt(20, os.path.join(out, "report.txt")) 68 | diff.to_md(20, os.path.join(out, "report.md")) 69 | diff.to_html(20, os.path.join(out, "report.html"), image_dir=".") 70 | 71 | ################# 72 | # /END taken from gftools-qa 73 | ################# 74 | 75 | class DiffenatorWorker(DiffWorkerBase): 76 | def __init__(self, logging, job, cache, persistence, queue, tmp_directory): 77 | self._workername = 'diffenator' 78 | super().__init__(logging, job, cache, persistence, queue, tmp_directory) 79 | self._answer.preparation_logs.append( 80 | 'Diffenator version {}'.format(diffenator.__version__)) 81 | 82 | def run(self): 83 | self._set_answer_timestamp('started') 84 | fonts = self._prepare(self._cache.get(self._job.cache_key).files, ['before', 'after']) 85 | # all_fonts = reduce(lambda a,b: a+b, fonts.values(),[]) 86 | all_files = [os.path.join(dp, f) for dp, dn, fn \ 87 | in os.walk(self._tmp_directory) for f in fn] 88 | self._log.debug('Files in Tempdir {}: {}'.format( 89 | self._tmp_directory, all_files)) 90 | 91 | self._log.info('entering run_diffenator …') 92 | # FIXME: should we collect stdout/stderr here??? 93 | run_diffenator(self._log, fonts['before'], fonts['after'], self._out_dir, DIFFENATOR_THRESHOLDS['normal']) 94 | self._log.info('DONE! docid: %s', self._job.docid) 95 | 96 | 97 | # The intention for main was ever only for debugging/profiling. 98 | # debug_run.py: 99 | # #!/usr/bin/env python3 100 | # 101 | # import logging 102 | # FORMAT = '%(asctime)s:%(name)s:%(levelname)s:%(message)s' 103 | # logging.basicConfig(format=FORMAT) 104 | # 105 | # import sys 106 | # print('python version:', sys.version) 107 | # 108 | # from worker.diffenator import main 109 | # main() 110 | # with memory profiling: 111 | # base/python$ mprof run debug_python.py 112 | def main(): 113 | import logging 114 | FORMAT = '%(asctime)s:%(name)s:%(levelname)s:%(message)s' 115 | logging.basicConfig(format=FORMAT) 116 | logger = logging.getLogger('DIFFENATOR_WORKER') 117 | import importlib 118 | wl = importlib.import_module('worker-launcher') 119 | setLoglevel = wl.setLoglevel 120 | getSetup = wl.getSetup 121 | 122 | setup = getSetup() 123 | setLoglevel(logger, setup.log_level) 124 | # DEBUG is a lot of output! 125 | # setLoglevel(logging.getLogger('fontdiffenator'), 'INFO') 126 | setLoglevel(logging.getLogger('fontdiffenator'), setup.log_level) 127 | logger.info('loglevel: ' + setup.log_level) 128 | 129 | fonts = {'before': [], 'after': []} 130 | 131 | tmp = '/var/python/debug_vollkorn' 132 | out_dir = os.path.join(tmp, 'result') 133 | os.mkdir(out_dir) 134 | # just collect the fonts 135 | for sub in fonts.keys(): 136 | dirname = os.path.join(tmp, sub) 137 | fonts[sub] = [os.path.join(dirname, filename)\ 138 | for filename in next(os.walk(dirname))[2]\ 139 | if filename.endswith('.ttf')] 140 | 141 | logger.info('fonts before:\n%s', '\n'.join(fonts['before'])) 142 | logger.info('fonts after:\n%s', '\n'.join(fonts['after'])) 143 | 144 | run_diffenator(logger, fonts['before'], fonts['after'], out_dir, DIFFENATOR_THRESHOLDS['normal']) 145 | -------------------------------------------------------------------------------- /containers/base/python/worker/storageclient.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import print_function, division, unicode_literals, absolute_import 3 | 4 | import sys 5 | import logging 6 | import time 7 | from protocolbuffers import messages_pb2_grpc 8 | from protocolbuffers.messages_pb2 import ( 9 | StorageItem 10 | ) 11 | from google.protobuf.any_pb2 import Any 12 | from collections import OrderedDict 13 | import grpc 14 | 15 | # currently unused 16 | def get_type_for_type_name(typeName, accepted_types): 17 | for Type in accepted_types: 18 | if Type.DESCRIPTOR.full_name == typeName: 19 | return Type 20 | raise KeyError('Type for name "{0}" not found.'.format(typeName)) 21 | 22 | # currently unused 23 | def get_type_from_any(any, accepted_types): 24 | return get_type_for_type_name(any.TypeName(), accepted_types) 25 | 26 | def unpack_any(any, Type): 27 | message = Type() 28 | if not any.Unpack(message): 29 | raise ValueError('any.Unpack Failed: expected type {}:"{}" doesn\'t match "{}".' 30 | .format(Type, Type.DESCRIPTOR.full_name, any.TypeName())) 31 | return message 32 | 33 | # currently unused 34 | def unpack_any_generic(any, accepted_types): 35 | Type = get_type_from_any(any, accepted_types) 36 | return unpack_any(any, Type) 37 | 38 | class RetriesExceeded(Exception): 39 | pass 40 | 41 | 42 | # Thanks @Bogdanp for this comment: 43 | # https://github.com/GoogleCloudPlatform/google-cloud-python/issues/2583#issuecomment-256026510 44 | MAX_TRIES_BY_CODE = { 45 | grpc.StatusCode.UNKNOWN: 6 # as reported in #56 46 | , grpc.StatusCode.INTERNAL: 1 47 | , grpc.StatusCode.UNAVAILABLE: 5 48 | , grpc.StatusCode.DEADLINE_EXCEEDED: 5 49 | } 50 | 51 | def backoff(f, *args, **kwds): 52 | tries = 0 53 | while True: 54 | tries += 1 55 | try: 56 | return f(*args, **kwds) 57 | # see https://github.com/googlefonts/fontbakery-dashboard/issues/56 58 | # Expecting a _Rendezvous 59 | # https://github.com/grpc/grpc/tree/master/src/python/grpcio/grpc/_channel.py 60 | except grpc.RpcError as error: 61 | code = error.code() 62 | if code not in MAX_TRIES_BY_CODE: 63 | raise error 64 | 65 | # There's no better way than to check the details, see #56 and #59 66 | if code == grpc.StatusCode.UNKNOWN and error.details() not in {'Stream removed'}: 67 | raise error 68 | 69 | if tries >= MAX_TRIES_BY_CODE[code]: 70 | raise RetriesExceeded(error) 71 | 72 | # retry in ... 73 | backoff = 0.0625 * 2 ** tries # 0.125, 0.25, 0.5, 1.0 74 | logging.warning('Exception in try #{0} backing off for {1} seconds ' 75 | 'until retry. Error: {2}'.format(tries, backoff, error)) 76 | time.sleep(backoff) 77 | 78 | class StorageClient(object): 79 | """ 80 | usage: 81 | 82 | from worker.storageclient import StorageClient 83 | from protocolbuffers.messages_pb2 import StorageKey, Files 84 | import sys 85 | client =StorageClient('localhost', 50051, Files) 86 | storageKey = StorageKey(key=sys.argv[1]) 87 | print('[GET] request:', storageKey) 88 | result = client.get(storageKey) 89 | print('[GET] result:', result) 90 | 91 | """ 92 | def __init__(self, host, port, ExpectedGetType): 93 | self._channel = grpc.insecure_channel('{}:{}'.format(host, port) 94 | , options=[ 95 | ('grpc.max_send_message_length', 80 * 1024 * 1024) 96 | , ('grpc.max_receive_message_length', 80 * 1024 * 1024) 97 | ] 98 | ) 99 | self._client = messages_pb2_grpc.StorageStub(self._channel) 100 | self.ExpectedGetType = ExpectedGetType 101 | 102 | def get(self, storageKey): 103 | any = backoff(self._client.Get, storageKey); 104 | return unpack_any(any, self.ExpectedGetType) 105 | 106 | def put (self, messages, ensure_answers_in_order=True): 107 | """ 108 | ensure_answers_in_order: bool, default True 109 | Using this to make sure we answer in the same order as we PUT. 110 | Could be done otherwise more memory efficient in some cases. 111 | However, user expectation are probably to get the answers in the 112 | same order as the messages and it would lead to subtle bugs 113 | otherwise. If ensure_answers_in_order is False the order of 114 | answers is not guaranteed to be the same order as messages, 115 | depending purely on the server implementation. 116 | """ 117 | if ensure_answers_in_order: 118 | result = OrderedDict() 119 | def make_storage_item(message, clientid): 120 | storage_item = StorageItem() 121 | any_message = Any() 122 | any_message.Pack(message) 123 | storage_item.payload.CopyFrom(any_message) 124 | if clientid is not None: 125 | storage_item.clientid = clientid 126 | if ensure_answers_in_order: 127 | result[clientid] = None 128 | return storage_item 129 | 130 | storage_items = (make_storage_item(message, str(index))\ 131 | for index, message in enumerate(messages)) 132 | 133 | for storageKey in self._client.Put(storage_items): 134 | if ensure_answers_in_order: 135 | result[storageKey.clientid] = storageKey 136 | else: 137 | yield storageKey 138 | 139 | if ensure_answers_in_order: 140 | for storageKey in result.values(): 141 | yield storageKey 142 | 143 | 144 | # Used for ad-hoc testing only! 145 | # Due to the mess of py3 imports of the protocolbuffers, called like 146 | # this: 147 | # containers/base/python$ python -c 'from worker.storageclient import main;main()' 148 | def main(): 149 | c = StorageClient('127.0.0.1', '3456', None) 150 | from google.protobuf.timestamp_pb2 import Timestamp 151 | from time import sleep 152 | 153 | def getCurrentTs(): 154 | sleep(.5) 155 | ts = Timestamp() 156 | ts.GetCurrentTime() 157 | print('generated another timestamp:', ts) 158 | return ts 159 | 160 | tss = (getCurrentTs() for _ in range(10)) 161 | for r in c.put(tss): 162 | print('result:', r); 163 | print('done!') 164 | -------------------------------------------------------------------------------- /containers/base/python/worker/worker_base.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import print_function, division, unicode_literals 3 | 4 | 5 | 6 | class WorkerError(Exception): 7 | pass 8 | 9 | class PreparationError(WorkerError): 10 | pass 11 | 12 | class WorkerBase: 13 | """In a sub-class define `__init__`, it's properties names are used 14 | for dependency injection. Have a look into worker-launcher to see how 15 | this is done. 16 | """ 17 | 18 | def JobType(): 19 | """JobType is expected to be a protocol buffers message constructor""" 20 | raise NotImplementedError('`JobType` method must be set as class ' 21 | 'property by sub-class.'); 22 | 23 | def run(self): 24 | """Run the job, exceptions will be caught and passed to`finalize`.""" 25 | raise NotImplementedError('`run` method must be implemented by sub-class.'); 26 | 27 | 28 | 29 | def finalize(self, tb_str, *exc): 30 | """If `run` failed and raised `tb_str` is a string of the traceback. 31 | If run finished regularly `tb_str` is None and there was no exception. 32 | 33 | return: If there was no exception, this is irrelevant. 34 | 35 | return: If there was an exception, return `Frue` if it has been dealt 36 | with in this method, otherwise return `False` to escalate the exception 37 | to the next level. 38 | """ 39 | 40 | raise NotImplementedError('`finalize` method must be implemented by sub-class.'); 41 | -------------------------------------------------------------------------------- /containers/base/update_protobufs.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | mkdir -p ./javascript/generated_modules/protocolbuffers; 4 | mkdir -p ./python/protocolbuffers; 5 | touch ./python/protocolbuffers/__init__.py 6 | 7 | # `grpc_tools_node_protoc` needs `npm install` grpc-tools is in dev dependencies 8 | PATH="$(pwd)/javascript/node_modules/.bin:$PATH" 9 | 10 | 11 | # created with: 12 | # $ python3 -m venv devvenv 13 | # $ source devvenv/bin/activate 14 | # $ pip install --upgrade pip 15 | # $ pip install -r dev-requirements.txt 16 | source devvenv/bin/activate 17 | 18 | 19 | 20 | pushd .; 21 | cd protocolbuffers; 22 | 23 | if [ -d ./gftools ]; then 24 | pushd .; cd gftools && git pull || exit 1; popd; 25 | else 26 | git clone --depth 1 -b main https://github.com/googlefonts/gftools.git || exit 1; 27 | fi 28 | cp gftools/Lib/gftools/fonts_public.proto . || exit 1 29 | 30 | echo "generating javascript protocolbuffers ..." 31 | 32 | # old, replaced by `grpc_tools_node_protoc` 33 | # protoc --js_out=import_style=commonjs,binary:../javascript/generated_modules/protocolbuffers *.proto; 34 | grpc_tools_node_protoc --js_out=import_style=commonjs,binary:../javascript/generated_modules/protocolbuffers/ \ 35 | --grpc_out=../javascript/generated_modules/protocolbuffers \ 36 | --plugin=protoc-gen-grpc=`which grpc_tools_node_protoc_plugin` \ 37 | *.proto 38 | 39 | ls ../javascript/generated_modules/protocolbuffers 40 | 41 | echo "generating python protocolbuffers ..." 42 | 43 | # protoc --python_out=../python/protocolbuffers *.proto 44 | python -m grpc_tools.protoc -I./ \ 45 | --python_out=../python/protocolbuffers \ 46 | --grpc_python_out=../python/protocolbuffers \ 47 | *.proto 48 | 49 | ls ../python/protocolbuffers 50 | 51 | echo 'DONE!' 52 | -------------------------------------------------------------------------------- /containers/rethinkdb/Dockerfile: -------------------------------------------------------------------------------- 1 | # This basically creates a hub.docker.com rethinkdb:2.3.6 but that one 2 | # is based on Debian 8 Jessie, while Debian 9 Stretch is current. 3 | # And the project doesn't support a stretch deb image :-/ 4 | # Then it mixes in a simplified yet inspired approach of rosskukulinski/rethinkdb-kubernetes 5 | FROM debian:buster 6 | 7 | RUN apt-get update && apt-get upgrade -y \ 8 | && apt-get install -y procps sudo mg git \ 9 | build-essential protobuf-compiler python \ 10 | libprotobuf-dev libcurl4-openssl-dev libboost-all-dev \ 11 | libncurses5-dev libjemalloc-dev wget m4 clang libssl-dev \ 12 | && rm -rf /var/lib/apt/lists/*; 13 | 14 | 15 | RUN mkdir -p /tmp/build \ 16 | && cd /tmp/build \ 17 | && wget https://download.rethinkdb.com/dist/rethinkdb-2.3.7.tgz \ 18 | && tar xf rethinkdb-2.3.7.tgz \ 19 | && cd rethinkdb-2.3.7 \ 20 | && ./configure --allow-fetch CXX=clang++ \ 21 | && make -j8 \ 22 | && make install \ 23 | && cd / \ 24 | && rm -rf /tmp/build; 25 | 26 | ENV TINI_VERSION v0.18.0 27 | ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini 28 | RUN chmod +x /tini 29 | ENTRYPOINT ["/tini", "--"] 30 | # Now use "args" instead of "command" in the yaml file pod specs!!! 31 | # because "command" in the yaml replaces the ENTRYPOINT in the container 32 | # and we want the PID 1 to be tini 33 | 34 | # some copy pasta 35 | # from https://github.com/rethinkdb/rethinkdb-dockerfiles/blob/master/jessie/2.3.6/Dockerfile 36 | VOLUME ["/data"] 37 | 38 | WORKDIR /data 39 | 40 | # process cluster webui 41 | EXPOSE 28015 29015 8080 42 | 43 | # now we can just do e.g.: 44 | # CMD ["rethinkdb", "--bind", "all"] 45 | # or in the pod spec: 46 | # args: ["rethinkdb", "--bind", "all"] 47 | # or actually in this case: 48 | # args: ["run.sh", "--bind", "all"] 49 | 50 | 51 | # stuff from https://github.com/helm/charts/blob/master/stable/rethinkdb/init/Dockerfile 52 | # at Latest commit 465c9fb on Jun 13, 2017 "rethinkdb: Initial Commit (#1018)" 53 | 54 | RUN apt-get update && \ 55 | apt-get install -yq curl && \ 56 | rm -rf /var/cache/apt/* && rm -rf /var/lib/apt/lists/* 57 | 58 | ADD http://stedolan.github.io/jq/download/linux64/jq /usr/bin/jq 59 | RUN chmod +x /usr/bin/jq 60 | 61 | COPY ./files/run.sh ./rethinkdb-probe/rethinkdb-probe / 62 | RUN chmod u+x /run.sh /rethinkdb-probe 63 | -------------------------------------------------------------------------------- /containers/rethinkdb/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The Kubernetes Authors All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | buildprobe: 16 | (cd ./rethinkdb-probe && ./build-probe.sh) 17 | -------------------------------------------------------------------------------- /containers/rethinkdb/files/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2015 The Kubernetes Authors All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o pipefail 18 | 19 | POD_NAMESPACE=${POD_NAMESPACE:-default} 20 | POD_IP=${POD_IP:-127.0.0.1} 21 | RETHINK_CLUSTER_SERVICE=${RETHINK_CLUSTER_SERVICE:-"rethinkdb"} 22 | POD_NAME=${POD_NAME:-"NO_POD_NAME"} 23 | RETHINKDB_PASSWORD=${RETHINKDB_PASSWORD:-"auto"} 24 | 25 | # Transform - to _ to comply with requirements 26 | SERVER_NAME=$(echo ${POD_NAME} | sed 's/-/_/g') 27 | 28 | echo "Using additional CLI flags: ${@}" 29 | echo "Pod IP: ${POD_IP}" 30 | echo "Pod namespace: ${POD_NAMESPACE}" 31 | echo "Using service name: ${RETHINK_CLUSTER_SERVICE}" 32 | echo "Using server name: ${SERVER_NAME}" 33 | 34 | echo "Checking for other nodes..." 35 | if [[ -n "${KUBERNETES_SERVICE_HOST}" ]]; then 36 | echo "Using endpoints to lookup other nodes..." 37 | URL="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}/api/v1/namespaces/${POD_NAMESPACE}/endpoints/${RETHINK_CLUSTER_SERVICE}" 38 | echo "Endpoint url: ${URL}" 39 | echo "Looking for IPs..." 40 | token=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) 41 | # try to pick up first different ip from endpoints 42 | IP=$(curl -s ${URL} --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt --header "Authorization: Bearer ${token}" \ 43 | | jq -s -r --arg h "${POD_IP}" '.[0].subsets | .[].addresses | [ .[].ip ] | map(select(. != $h)) | .[0]') || exit 1 44 | [[ "${IP}" == null ]] && IP="" 45 | JOIN_ENDPOINTS="${IP}" 46 | fi 47 | 48 | # xargs echo removes extra spaces before/after 49 | # tr removes extra spaces in the middle 50 | JOIN_ENDPOINTS=$(echo ${JOIN_ENDPOINTS} | xargs echo | tr -s ' ') 51 | 52 | if [ -n "${JOIN_ENDPOINTS}" ]; then 53 | echo "Found other nodes: ${JOIN_ENDPOINTS}" 54 | 55 | # Now, transform join endpoints into --join ENDPOINT:29015 56 | # Put port after each 57 | JOIN_ENDPOINTS=$(echo ${JOIN_ENDPOINTS} | sed -r 's/([0-9.])+/&:29015/g') 58 | 59 | # Put --join before each 60 | JOIN_ENDPOINTS=$(echo ${JOIN_ENDPOINTS} | sed -e 's/^\|[ ]/&--join /g') 61 | else 62 | echo "No other nodes detected, will be a single instance." 63 | if [ -n "$PROXY" ]; then 64 | echo "Cannot start in proxy mode without endpoints." 65 | exit 1 66 | fi 67 | fi 68 | 69 | if [[ -n "${PROXY}" ]]; then 70 | echo "Starting in proxy mode" 71 | set -x 72 | exec rethinkdb \ 73 | proxy \ 74 | --initial-password ${RETHINKDB_PASSWORD} \ 75 | --canonical-address ${POD_IP} \ 76 | ${JOIN_ENDPOINTS} \ 77 | ${@} 78 | else 79 | set -x 80 | exec rethinkdb \ 81 | --server-name ${SERVER_NAME} \ 82 | --initial-password ${RETHINKDB_PASSWORD} \ 83 | --canonical-address ${POD_IP} \ 84 | ${JOIN_ENDPOINTS} \ 85 | ${@} 86 | fi 87 | -------------------------------------------------------------------------------- /containers/rethinkdb/rethinkdb-probe/Dockerfile.build: -------------------------------------------------------------------------------- 1 | FROM golang:1.12.5 2 | MAINTAINER Chris Dornsife 3 | 4 | # Build container to have a consistent go build environment 5 | 6 | COPY . /go/src/rethinkdb-probe 7 | WORKDIR /go/src/rethinkdb-probe 8 | 9 | RUN go get ./... \ 10 | && CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-w -s' -o /target/rethinkdb-probe . 11 | -------------------------------------------------------------------------------- /containers/rethinkdb/rethinkdb-probe/build-probe.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Author: Chris Dornsife chris@dornsife.com 4 | # This will run a build container and extract a binary. 5 | # This doesn't require the use of a mount so it can be used 6 | # in a build pipeline such as bitbucket. 7 | PROJ=rethinkdb-probe 8 | 9 | HASH=`date +%s` 10 | BUILD_NAME=${PROJ}-build-${HASH} 11 | 12 | docker build -t ${BUILD_NAME} -f ./Dockerfile.build . || exit 1 13 | docker create --name ${BUILD_NAME} ${BUILD_NAME} /bin/true || exit 1 14 | docker cp ${BUILD_NAME}:/target/$PROJ ./$PROJ || exit 1 15 | docker rm ${BUILD_NAME} || exit 1 16 | docker rmi -f ${BUILD_NAME} || exit 1 17 | 18 | chmod +x ./$PROJ 19 | -------------------------------------------------------------------------------- /containers/rethinkdb/rethinkdb-probe/probe.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | r "gopkg.in/gorethink/gorethink.v2" 7 | ) 8 | 9 | 10 | func main() { 11 | 12 | url := os.Getenv("RETHINKDB_URL") 13 | password := os.Getenv("RETHINKDB_PASSWORD") 14 | 15 | if url == "" { 16 | url = "localhost:28015" 17 | } 18 | 19 | session, err := r.Connect(r.ConnectOpts{ 20 | Address: url, 21 | Database: "rethinkdb", 22 | Username: "admin", 23 | Password: password, 24 | }) 25 | 26 | if err != nil { 27 | log.Fatalln(err.Error()) 28 | } 29 | 30 | res, err := r.Table("server_status").Pluck("id", "name").Run(session) 31 | if err != nil { 32 | log.Fatalln(err.Error()) 33 | } 34 | defer res.Close() 35 | 36 | if res.IsNil() { 37 | log.Fatalln("no server status results found") 38 | } 39 | 40 | log.Printf("A-OK!") 41 | 42 | } 43 | -------------------------------------------------------------------------------- /containers/rethinkdb/rethinkdb-probe/rethinkdb-probe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/containers/rethinkdb/rethinkdb-probe/rethinkdb-probe -------------------------------------------------------------------------------- /docs/nodes-diagram-v2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlefonts/fontbakery-dashboard/1f949a898007c961ca5adef37337b6236580daf6/docs/nodes-diagram-v2.pdf -------------------------------------------------------------------------------- /docs/the-report-format.md: -------------------------------------------------------------------------------- 1 | # The Report Format 2 | 3 | The report format will be described as JSON here. Though it is currently 4 | a rethinkDB document. Some of the rationale and intended usage is 5 | documented here. 6 | 7 | 8 | ## Top Level Keys 9 | 10 | ### `` 11 | 12 | ``` 13 | { 14 | // always there 15 | "created": // document creation time 16 | , "id": // the unique document id in the database(-table?) 17 | // after processing has started 18 | , "started": // processing start time 19 | , "preparation_logs": // lines of logs of the dispatcher preparation 20 | , "execution_order" // -Keys in fontbakeries order 21 | , "iterargs": // keys the names of the iterargs; values: 22 | , "jobs": // jobs metadata s... TODO: explain why and rationale 23 | , "tests": // keys: ; values: test result s 24 | // in case of an exception 25 | , "finished": // job end time 26 | , "exception": // exception and traceback 27 | } 28 | 29 | There's no `finished` for regular execution. When all jobs have a `finished` 30 | date (all tests must have registered their statuses), `finished` equals 31 | the highest `finished` of all jobs. 32 | 33 | ``` 34 | 35 | ### `` 36 | 37 | Not sure why tracking jobs this way. Though, we can see when a job fails 38 | finally or when it never ran. For future jobs that answer with high latency 39 | (because humans need to work them) This could be the perfect place to look 40 | where/how to start them. Such a job can itself yield multiple tests, which 41 | seems like a good, further way of structuring, if needed. The job must somehow 42 | register its tests in the spec. The dispatcher has to know how to group these 43 | jobs. May be a big effort, but would integrate well. 44 | 45 | ``` 46 | { 47 | // always there 48 | "created": // job creation time; is ~ document creation time 49 | , "id": // unique id and index in jobs 50 | // after processing has started 51 | , "started": // processing start time 52 | // after the job ended (also if ended exceptionally) 53 | , "finished": // job end time 54 | // in case of an exception 55 | , "exception": // exception and traceback 56 | ``` 57 | 58 | ### `` 59 | 60 | Do we need more info here? started, ended, microtime ended-started? 61 | When finished: aggregeate results? (we could get that probably with a 62 | good reql query). Also, if we plan to do overrides, we'll have a lot 63 | of intepretation going on anyways. 64 | Not sure if we should break the SSOT here. Maybe we'll do when there's a 65 | good reason to do so (speed, code is not DRY enough). 66 | 67 | We could put the order index in here! Why not? We'd need to count tests that 68 | have a status to see if everything is finished. 69 | 70 | ``` 71 | { 72 | "job_id": 0 73 | , "statuses": [ 74 | { 75 | "message": "/tmp/tmpV4Hfn9/RobotoSlab-Bold.ttf is named canonically." , 76 | "status": "PASS" 77 | } 78 | ] 79 | } 80 | ``` 81 | 82 | 83 | 84 | tests as an array vs. an dict: 85 | 86 | pro array: 87 | - Append is probably faster than the rethinkdb `merge` equivalent for a dict 88 | - With append we have a way to see when a job runs multiple times* (for whatever reason). 89 | It may be better to factor in possible failure! A list with append could 90 | be interpreted in any way. 91 | 92 | pro dict: 93 | - When the number of keys equals the length of "execution_order" the test is 94 | finished. With append, failing jobs could make the list longer than the 95 | "execution_order" and we could still miss some tests. RethinkDB has an easy 96 | "count" function to count the number of key/value pairs in an object! 97 | - we already have execution_order to store the order, reading tests by key seems 98 | natural. order of being logged is not essential. 99 | 100 | 101 | 102 | 103 | 104 | * A job runs multiple times: 105 | 106 | We now have good error handling, so the job wouldn't crash. But the default 107 | behavior of AMPQ is to re-queue a message when the receiver dies and Kubernetes 108 | restarts a pod that has died. The rationale is, that such an error may be 109 | caused by missing external resources (e.g. Database) and these may become 110 | available later. Thus, If our job runs half way through, then dies fatally, 111 | it may be better to have it resubmit test results instead of not finishing 112 | the ever. 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | -------------------------------------------------------------------------------- /kubernetes/gcloud-fontbakery-api.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fontbakery-api 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | run: fontbakery-api 10 | template: 11 | metadata: 12 | labels: 13 | run: fontbakery-api 14 | spec: 15 | nodeSelector: 16 | cloud.google.com/gke-nodepool: default-pool 17 | containers: 18 | - name: fontbakery-api 19 | image: gcr.io/fontbakery-168509/base-javascript:20 20 | env: 21 | - name: FONTBAKERY_LOG_LEVEL 22 | value: "DEBUG" 23 | - name: WEB_SERVER_COOKIE_SECRET 24 | valueFrom: 25 | secretKeyRef: 26 | name: external-resources 27 | key: web-server-cookie-secret 28 | - name: RETHINKDB_PASSWORD 29 | valueFrom: 30 | secretKeyRef: 31 | name: external-resources 32 | key: rethinkdb-password 33 | ports: 34 | - containerPort: 3000 35 | # demanded by https://cloud.google.com/kubernetes-engine/docs/concepts/ingress 36 | # we use just the standard client response here, which has a http 200 status 37 | readinessProbe: 38 | httpGet: 39 | path: / 40 | port: 3000 41 | # debugger chrome://inspect port if active 42 | # - containerPort: 9229 43 | workingDir: /var/javascript 44 | # CAUTION! 45 | # command: ["node", "--inspect=0.0.0.0:9229", "node/api.js"] 46 | command: ["npm", "start"] 47 | --- 48 | apiVersion: cloud.google.com/v1beta1 49 | kind: BackendConfig 50 | metadata: 51 | name: fontbakery-api-backendconfig 52 | spec: 53 | timeoutSec: 900 54 | # TODO: need this for socketIO and multiple fontbakery-api servers 55 | # needs a "VPC-native cluster" (--enable-ip-alias) but the current 56 | # cluster was not created like this and can't be changed on the fly 57 | # apparently. 58 | # see: https://cloud.google.com/kubernetes-engine/docs/how-to/configure-backend-service#setting_generated_cookie_affinity 59 | # sessionAffinity: 60 | # affinityType: "GENERATED_COOKIE" # OR "CLIENT_IP" 61 | --- 62 | apiVersion: v1 63 | kind: Service 64 | metadata: 65 | name: fontbakery-api 66 | labels: 67 | run: fontbakery-api 68 | annotations: 69 | # see sessionAffinity in fontbakery-api-backendconfig 70 | # cloud.google.com/neg: '{"ingress": true}' 71 | beta.cloud.google.com/backend-config: '{"ports": {"80":"fontbakery-api-backendconfig"}}' 72 | spec: 73 | type: NodePort 74 | ports: 75 | - port: 80 76 | targetPort: 3000 77 | protocol: TCP 78 | name: http 79 | #- port: 443 80 | # targetPort: 3000 81 | # protocol: TCP 82 | # name: https 83 | selector: 84 | run: fontbakery-api 85 | --- 86 | apiVersion: networking.gke.io/v1beta1 87 | kind: ManagedCertificate 88 | metadata: 89 | name: fontbakery-graphicore-de 90 | spec: 91 | domains: 92 | - fontbakery.graphicore.de 93 | --- 94 | apiVersion: extensions/v1beta1 95 | kind: Ingress 96 | metadata: 97 | name: fontbakery-api-ingress 98 | annotations: 99 | kubernetes.io/ingress.global-static-ip-name: fontbakery-dashboard # 34.96.110.16 100 | networking.gke.io/managed-certificates: fontbakery-graphicore-de 101 | # only https: 102 | kubernetes.io/ingress.allow-http: "false" 103 | spec: 104 | rules: 105 | - host: fontbakery.graphicore.de 106 | http: 107 | paths: 108 | - backend: 109 | serviceName: fontbakery-api 110 | servicePort: http 111 | -------------------------------------------------------------------------------- /kubernetes/gcloud-fontbakery-dispatcher.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fontbakery-dispatcher 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | component: fontbakery-dispatcher 10 | template: 11 | metadata: 12 | labels: 13 | component: fontbakery-dispatcher 14 | spec: 15 | containers: 16 | - name: fontbakery-dispatcher 17 | image: gcr.io/fontbakery-168509/base-javascript:20 18 | env: 19 | - name: FONTBAKERY_LOG_LEVEL 20 | value: "DEBUG" 21 | - name: FRONTEND_BASE_URL 22 | valueFrom: 23 | configMapKeyRef: 24 | name: env-config 25 | key: frontend-base-url 26 | - name: DISPATCHER_MANAGER_SECRET 27 | valueFrom: 28 | secretKeyRef: 29 | name: external-resources 30 | key: dispatcher-manager-secret 31 | - name: RETHINKDB_PASSWORD 32 | valueFrom: 33 | secretKeyRef: 34 | name: external-resources 35 | key: rethinkdb-password 36 | ports: 37 | - containerPort: 50051 38 | workingDir: /var/javascript 39 | command: ["node", "node/dispatcher/DispatcherProcessManager.js", "-p", "50051"] 40 | # command: ["node", "--inspect=0.0.0.0:9229", "node/dispatcher/DispatcherProcessManager.js", "-p", "50051"] 41 | --- 42 | apiVersion: v1 43 | kind: Service 44 | metadata: 45 | name: fontbakery-dispatcher 46 | labels: 47 | component: fontbakery-dispatcher 48 | spec: 49 | ports: 50 | - port: 50051 51 | targetPort: 50051 52 | selector: 53 | component: fontbakery-dispatcher 54 | -------------------------------------------------------------------------------- /kubernetes/gcloud-fontbakery-github-auth.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fontbakery-github-auth 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | component: fontbakery-github-auth 10 | template: 11 | metadata: 12 | labels: 13 | component: fontbakery-github-auth 14 | spec: 15 | containers: 16 | - name: fontbakery-github-auth 17 | image: gcr.io/fontbakery-168509/base-javascript:20 18 | env: 19 | - name: FONTBAKERY_LOG_LEVEL 20 | value: "DEBUG" 21 | - name: GITHUB_OAUTH_CLIENT_ID 22 | valueFrom: 23 | secretKeyRef: 24 | name: external-resources 25 | key: github-oauth-client-id 26 | - name: GITHUB_OAUTH_CLIENT_SECRET 27 | valueFrom: 28 | secretKeyRef: 29 | name: external-resources 30 | key: github-oauth-client-secret 31 | - name: GITHUB_AUTH_ENGINEERS 32 | valueFrom: 33 | secretKeyRef: 34 | name: external-resources 35 | key: github-auth-engineers 36 | ports: 37 | - containerPort: 50051 38 | workingDir: /var/javascript 39 | command: ["node", "node/GitHubAuthServer.js", "-p", "50051"] 40 | --- 41 | apiVersion: v1 42 | kind: Service 43 | metadata: 44 | name: fontbakery-github-auth 45 | labels: 46 | component: fontbakery-github-auth 47 | spec: 48 | ports: 49 | - port: 50051 50 | targetPort: 50051 51 | selector: 52 | component: fontbakery-github-auth 53 | -------------------------------------------------------------------------------- /kubernetes/gcloud-fontbakery-github-operations.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fontbakery-github-operations 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | component: fontbakery-github-operations 10 | template: 11 | metadata: 12 | labels: 13 | component: fontbakery-github-operations 14 | spec: 15 | containers: 16 | - name: fontbakery-github-operations 17 | image: gcr.io/fontbakery-168509/base-javascript:20 18 | env: 19 | - name: FONTBAKERY_LOG_LEVEL 20 | value: "DEBUG" 21 | - name: GITHUB_API_TOKEN 22 | valueFrom: 23 | secretKeyRef: 24 | name: external-resources 25 | key: github-api-token 26 | ports: 27 | - containerPort: 50051 28 | workingDir: /var/javascript 29 | command: ["node", "node/GitHubOperationsServer.js", "-p", "50051"] 30 | --- 31 | apiVersion: v1 32 | kind: Service 33 | metadata: 34 | name: fontbakery-github-operations 35 | labels: 36 | component: fontbakery-github-operations 37 | spec: 38 | ports: 39 | - port: 50051 40 | targetPort: 50051 41 | selector: 42 | component: fontbakery-github-operations 43 | -------------------------------------------------------------------------------- /kubernetes/gcloud-fontbakery-init-workers.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fontbakery-init-workers 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | component: fontbakery-init-workers 10 | template: 11 | metadata: 12 | labels: 13 | component: fontbakery-init-workers 14 | role: init-workers 15 | spec: 16 | containers: 17 | - name: fontbakery-init-workers 18 | image: gcr.io/fontbakery-168509/base-javascript:20 19 | env: 20 | - name: FONTBAKERY_LOG_LEVEL 21 | value: "DEBUG" 22 | - name: ENVIRONMENT_VERSION 23 | valueFrom: 24 | configMapKeyRef: 25 | name: env-config 26 | key: ENVIRONMENT_VERSION 27 | - name: RETHINKDB_PASSWORD 28 | valueFrom: 29 | secretKeyRef: 30 | name: external-resources 31 | key: rethinkdb-password 32 | ports: 33 | - containerPort: 50051 34 | workingDir: /var/javascript 35 | command: ["node", "node/InitWorkers.js", "-p", "50051"] 36 | #command: ["node", "--inspect=0.0.0.0:9222", "node/InitWorkers.js", "-p", "50051"] 37 | --- 38 | apiVersion: v1 39 | kind: Service 40 | metadata: 41 | name: fontbakery-init-workers 42 | labels: 43 | component: fontbakery-init-workers 44 | spec: 45 | ports: 46 | - port: 50051 47 | targetPort: 50051 48 | selector: 49 | component: fontbakery-init-workers 50 | role: init-workers 51 | -------------------------------------------------------------------------------- /kubernetes/gcloud-fontbakery-manifest-csvupstream.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fontbakery-manifest-csvupstream 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | component: fontbakery-manifest-csvupstream 10 | template: 11 | metadata: 12 | labels: 13 | component: fontbakery-manifest-csvupstream 14 | role: manifest 15 | spec: 16 | nodeSelector: 17 | cloud.google.com/gke-nodepool: default-pool 18 | containers: 19 | - name: fontbakery-manifest-csvupstream 20 | image: gcr.io/fontbakery-168509/base-javascript:20 21 | env: 22 | - name: FONTBAKERY_LOG_LEVEL 23 | value: "DEBUG" 24 | #- name: DEVEL_FAMILY_WHITELIST 25 | # value: "[\"Rosario\", \"Amiri\", \"Pacifico\", \"Astloch\"]" 26 | - name: CSV_SHEET_URL_SANDBOX 27 | value: https://docs.google.com/spreadsheets/d/1ODnp-yRYw1LrI3RTX-VZZsigPPieviE954sOsrlcx5o/pub?gid=0&single=true&output=csv 28 | - name: CSV_SHEET_URL_UPSTREAM 29 | value: https://docs.google.com/spreadsheets/d/1ampzD9veEdrwUMkOAJkMNkftqtv1jEygiPR0wZ6eNl8/pub?gid=0&single=true&output=csv 30 | workingDir: /var/javascript 31 | command: ["node", "node/manifestSources/CSVSpreadsheet.js", "-p", "50051"] 32 | # command: ["node", "--inspect=0.0.0.0:9229", "node/manifestSources/CSVSpreadsheet.js", "-p", "50051"] 33 | --- 34 | apiVersion: v1 35 | kind: Service 36 | metadata: 37 | name: fontbakery-manifest-csvupstream 38 | labels: 39 | component: fontbakery-manifest-csvupstream 40 | spec: 41 | ports: 42 | - port: 50051 43 | targetPort: 50051 44 | selector: 45 | component: fontbakery-manifest-csvupstream 46 | role: manifest 47 | -------------------------------------------------------------------------------- /kubernetes/gcloud-fontbakery-manifest-gfapi.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fontbakery-manifest-gfapi 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | component: fontbakery-manifest-gfapi 10 | template: 11 | metadata: 12 | labels: 13 | component: fontbakery-manifest-gfapi 14 | role: manifest 15 | spec: 16 | nodeSelector: 17 | cloud.google.com/gke-nodepool: default-pool 18 | containers: 19 | - name: fontbakery-manifest-gfapi 20 | image: gcr.io/fontbakery-168509/base-javascript:20 21 | env: 22 | - name: FONTBAKERY_LOG_LEVEL 23 | value: "DEBUG" 24 | #- name: DEVEL_FAMILY_WHITELIST 25 | # value: "[\"Rosario\", \"Amiri\", \"Pacifico\", \"Astloch\"]" 26 | - name: GOOGLE_API_KEY 27 | valueFrom: 28 | secretKeyRef: 29 | name: external-resources 30 | key: google-api-key 31 | workingDir: /var/javascript 32 | command: ["node", "node/manifestSources/GoogleFonts.js", "-p", "50051"] 33 | --- 34 | apiVersion: v1 35 | kind: Service 36 | metadata: 37 | name: fontbakery-manifest-gfapi 38 | labels: 39 | component: fontbakery-manifest-gfapi 40 | spec: 41 | ports: 42 | - port: 50051 43 | targetPort: 50051 44 | selector: 45 | component: fontbakery-manifest-gfapi 46 | role: manifest 47 | -------------------------------------------------------------------------------- /kubernetes/gcloud-fontbakery-manifest-githubgf.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fontbakery-manifest-githubgf 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | component: fontbakery-manifest-githubgf 10 | template: 11 | metadata: 12 | labels: 13 | component: fontbakery-manifest-githubgf 14 | role: manifest 15 | spec: 16 | nodeSelector: 17 | cloud.google.com/gke-nodepool: default-pool 18 | containers: 19 | - name: fontbakery-manifest-githubgf 20 | image: gcr.io/fontbakery-168509/base-javascript:20 21 | env: 22 | - name: FONTBAKERY_LOG_LEVEL 23 | value: "DEBUG" 24 | #- name: DEVEL_FAMILY_WHITELIST 25 | # value: "[\"Seoul Hangang Condensed\"]" 26 | - name: GITHUB_API_TOKEN 27 | valueFrom: 28 | secretKeyRef: 29 | name: external-resources 30 | key: github-api-token 31 | workingDir: /var/javascript 32 | command: ["node", "node/manifestSources/Git.js", "-p", "50051"] 33 | --- 34 | apiVersion: v1 35 | kind: Service 36 | metadata: 37 | name: fontbakery-manifest-githubgf 38 | labels: 39 | component: fontbakery-manifest-githubgf 40 | spec: 41 | ports: 42 | - port: 50051 43 | targetPort: 50051 44 | selector: 45 | component: fontbakery-manifest-githubgf 46 | role: manifest 47 | -------------------------------------------------------------------------------- /kubernetes/gcloud-fontbakery-manifest-main.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fontbakery-manifest-main 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | component: fontbakery-manifest-main 10 | template: 11 | metadata: 12 | labels: 13 | component: fontbakery-manifest-main 14 | role: manifest 15 | spec: 16 | nodeSelector: 17 | cloud.google.com/gke-nodepool: default-pool 18 | containers: 19 | - name: fontbakery-manifest-main 20 | image: gcr.io/fontbakery-168509/base-javascript:20 21 | env: 22 | - name: FONTBAKERY_LOG_LEVEL 23 | value: "DEBUG" 24 | - name: RETHINKDB_PASSWORD 25 | valueFrom: 26 | secretKeyRef: 27 | name: external-resources 28 | key: rethinkdb-password 29 | workingDir: /var/javascript 30 | command: ["node", "node/ManifestMain.js"] 31 | -------------------------------------------------------------------------------- /kubernetes/gcloud-fontbakery-reports.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fontbakery-reports 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | component: fontbakery-reports 10 | template: 11 | metadata: 12 | labels: 13 | component: fontbakery-reports 14 | role: reportsserver 15 | spec: 16 | nodeSelector: 17 | cloud.google.com/gke-nodepool: default-pool 18 | containers: 19 | - name: fontbakery-reports 20 | image: gcr.io/fontbakery-168509/base-javascript:20 21 | env: 22 | - name: FONTBAKERY_LOG_LEVEL 23 | value: "DEBUG" 24 | - name: RETHINKDB_PASSWORD 25 | valueFrom: 26 | secretKeyRef: 27 | name: external-resources 28 | key: rethinkdb-password 29 | ports: 30 | - containerPort: 50051 31 | workingDir: /var/javascript 32 | command: ["node", "node/ReportsServer.js", "-p", "50051"] 33 | # command: ["node", "--inspect=0.0.0.0:9229", "node/ReportsServer.js", "-p", "50051"] 34 | --- 35 | apiVersion: v1 36 | kind: Service 37 | metadata: 38 | name: fontbakery-reports 39 | labels: 40 | component: fontbakery-reports 41 | spec: 42 | ports: 43 | - port: 50051 44 | targetPort: 50051 45 | selector: 46 | component: fontbakery-reports 47 | role: reportsserver 48 | -------------------------------------------------------------------------------- /kubernetes/gcloud-fontbakery-storage-cache.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fontbakery-storage-cache 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | component: fontbakery-storage-cache 10 | template: 11 | metadata: 12 | labels: 13 | component: fontbakery-storage-cache 14 | role: messagecache 15 | spec: 16 | nodeSelector: 17 | cloud.google.com/gke-nodepool: default-pool 18 | containers: 19 | - name: fontbakery-storage-cache 20 | image: gcr.io/fontbakery-168509/base-javascript:20 21 | env: 22 | - name: FONTBAKERY_LOG_LEVEL 23 | value: "DEBUG" 24 | ports: 25 | - containerPort: 50051 26 | resources: 27 | limits: 28 | cpu: 1 29 | memory: 5Gi 30 | requests: 31 | cpu: .5 32 | memory: 2Gi 33 | workingDir: /var/javascript 34 | command: ["node", "node/CacheServer.js", "-p", "50051"] 35 | --- 36 | apiVersion: v1 37 | kind: Service 38 | metadata: 39 | name: fontbakery-storage-cache 40 | labels: 41 | component: fontbakery-storage-cache 42 | spec: 43 | ports: 44 | - port: 50051 45 | targetPort: 50051 46 | selector: 47 | component: fontbakery-storage-cache 48 | role: messagecache 49 | -------------------------------------------------------------------------------- /kubernetes/gcloud-fontbakery-storage-persistence.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fontbakery-storage-persistence 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | component: fontbakery-storage-persistence 10 | template: 11 | metadata: 12 | labels: 13 | component: fontbakery-storage-persistence 14 | role: messagecache 15 | spec: 16 | nodeSelector: 17 | cloud.google.com/gke-nodepool: default-pool 18 | affinity: 19 | nodeAffinity: 20 | requiredDuringSchedulingIgnoredDuringExecution: 21 | nodeSelectorTerms: 22 | - matchExpressions: 23 | - key: "failure-domain.beta.kubernetes.io/zone" 24 | operator: In 25 | values: ["us-central1-a"] # the storage volume must be here! 26 | containers: 27 | - name: fontbakery-storage-persistence 28 | image: gcr.io/fontbakery-168509/base-javascript:20 29 | env: 30 | - name: FONTBAKERY_LOG_LEVEL 31 | value: "DEBUG" 32 | - name: FONTBAKERY_PERSISTENT_DATA_DIR 33 | value: "/data/persistence_server" 34 | ports: 35 | - containerPort: 50051 36 | # resources: 37 | # limits: 38 | # cpu: 1 39 | # memory: 5Gi 40 | # requests: 41 | # cpu: .5 42 | # memory: 2Gi 43 | volumeMounts: 44 | - mountPath: /data 45 | name: storage 46 | workingDir: /var/javascript 47 | command: ["node", "node/PersistenceServer.js", "-p", "50051"] 48 | # command: ["node", "--inspect=0.0.0.0:9229", "node/PersistenceServer.js", "-p", "50051"] 49 | volumes: 50 | - gcePersistentDisk: 51 | # This GCE PD must already exist. 52 | fsType: ext4 53 | pdName: storage-persistence-ssd-1 54 | name: storage 55 | --- 56 | apiVersion: v1 57 | kind: Service 58 | metadata: 59 | name: fontbakery-storage-persistence 60 | labels: 61 | component: fontbakery-storage-persistence 62 | spec: 63 | ports: 64 | - port: 50051 65 | targetPort: 50051 66 | selector: 67 | component: fontbakery-storage-persistence 68 | role: messagecache 69 | -------------------------------------------------------------------------------- /kubernetes/gcloud-rabbitmq.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | labels: 5 | component: rabbitmq 6 | name: rabbitmq 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | component: rabbitmq 12 | template: 13 | metadata: 14 | labels: 15 | app: taskQueue 16 | component: rabbitmq 17 | spec: 18 | nodeSelector: 19 | cloud.google.com/gke-nodepool: default-pool 20 | containers: 21 | - image: rabbitmq:3-management 22 | name: rabbitmq 23 | ports: 24 | - containerPort: 5672 25 | env: 26 | - name: RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS 27 | value: "-rabbit heartbeat 1200" 28 | resources: 29 | limits: 30 | cpu: 1 31 | memory: 2048Mi 32 | requests: 33 | cpu: .5 34 | memory: 1024Mi 35 | # Setting these must be reflected in the clients! 36 | # env: 37 | # - name: RABBITMQ_DEFAULT_USER 38 | # value: "admin" 39 | # - name: RABBITMQ_DEFAULT_PASS 40 | # value: "admin" 41 | --- 42 | apiVersion: v1 43 | kind: Service 44 | metadata: 45 | name: rabbitmq-management 46 | labels: 47 | component: rabbitmq 48 | spec: 49 | ports: 50 | - port: 8888 51 | targetPort: 15672 52 | name: http 53 | selector: 54 | component: rabbitmq 55 | --- 56 | apiVersion: v1 57 | kind: Service 58 | metadata: 59 | name: rabbitmq-service 60 | labels: 61 | component: rabbitmq 62 | spec: 63 | ports: 64 | - port: 5672 65 | selector: 66 | app: taskQueue 67 | component: rabbitmq 68 | -------------------------------------------------------------------------------- /kubernetes/minikube-fontbakery-api.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fontbakery-api 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | run: fontbakery-api 10 | template: 11 | metadata: 12 | labels: 13 | run: fontbakery-api 14 | spec: 15 | containers: 16 | - name: fontbakery-api 17 | image: fontbakery/base-javascript:1 18 | env: 19 | - name: FONTBAKERY_LOG_LEVEL 20 | value: "DEBUG" 21 | - name: WEB_SERVER_COOKIE_SECRET 22 | valueFrom: 23 | secretKeyRef: 24 | name: external-resources 25 | key: web-server-cookie-secret 26 | - name: RETHINKDB_PASSWORD 27 | valueFrom: 28 | secretKeyRef: 29 | name: external-resources 30 | key: rethinkdb-password 31 | ports: 32 | - containerPort: 3000 33 | # NOT IN PRODUCTION 34 | - containerPort: 9229 35 | workingDir: /var/javascript 36 | # **NEVER** DO THIS IN PRODUCTION! 37 | # alias kf="kubectl -n fontbakery" 38 | # kf get pods 39 | # kf port-forward fontbakery-api-12334-123 9229:9229 40 | # open chrome and go to chrome://inspect 41 | command: ["node", "--inspect=0.0.0.0:9229", "node/api.js"] 42 | # PRODUCTION: 43 | # command: ["npm", "start"] 44 | # depends directly on rabbitmq/amqp 45 | # wait until rabbitmq can be reached before starting the actual container 46 | initContainers: 47 | - name: init-wait-for-rabbitmq 48 | image: busybox 49 | command: 50 | - sh 51 | - "-c" 52 | - > 53 | until wget -q -O - 54 | http://guest:guest@$RABBITMQ_MANAGEMENT_SERVICE_HOST:$RABBITMQ_MANAGEMENT_SERVICE_PORT/api/aliveness-test/%2F; 55 | do echo [`date`] waiting for rabbitmq: 10s; 56 | sleep 10; 57 | done; 58 | --- 59 | apiVersion: v1 60 | kind: Service 61 | metadata: 62 | name: fontbakery-api 63 | labels: 64 | run: fontbakery-api 65 | spec: 66 | type: LoadBalancer 67 | ports: 68 | - port: 80 69 | targetPort: 3000 70 | protocol: TCP 71 | name: http 72 | selector: 73 | run: fontbakery-api 74 | -------------------------------------------------------------------------------- /kubernetes/minikube-fontbakery-dispatcher.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fontbakery-dispatcher 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | component: fontbakery-dispatcher 10 | template: 11 | metadata: 12 | labels: 13 | component: fontbakery-dispatcher 14 | spec: 15 | containers: 16 | - name: fontbakery-dispatcher 17 | image: fontbakery/base-javascript:1 18 | env: 19 | - name: FONTBAKERY_LOG_LEVEL 20 | value: "DEBUG" 21 | - name: FRONTEND_BASE_URL 22 | valueFrom: 23 | configMapKeyRef: 24 | name: env-config 25 | key: frontend-base-url 26 | - name: DISPATCHER_MANAGER_SECRET 27 | valueFrom: 28 | secretKeyRef: 29 | name: external-resources 30 | key: dispatcher-manager-secret 31 | - name: RETHINKDB_PASSWORD 32 | valueFrom: 33 | secretKeyRef: 34 | name: external-resources 35 | key: rethinkdb-password 36 | ports: 37 | - containerPort: 50051 38 | workingDir: /var/javascript 39 | # command: ["node", "node/dispatcher/DispatcherProcessManager.js", "-p", "50051"] 40 | # kf port-forward fontbakery-dispatcher-12334-123 9229:9229 41 | command: ["node", "--inspect=0.0.0.0:9229", "node/dispatcher/DispatcherProcessManager.js", "-p", "50051"] 42 | # depends directly on rabbitmq/amqp 43 | # wait until rabbitmq can be reached before starting the actual container 44 | initContainers: 45 | - name: init-wait-for-rabbitmq 46 | image: busybox 47 | command: 48 | - sh 49 | - "-c" 50 | - > 51 | until wget -q -O - 52 | http://guest:guest@$RABBITMQ_MANAGEMENT_SERVICE_HOST:$RABBITMQ_MANAGEMENT_SERVICE_PORT/api/aliveness-test/%2F; 53 | do echo [`date`] waiting for rabbitmq: 10s; 54 | sleep 10; 55 | done; 56 | --- 57 | apiVersion: v1 58 | kind: Service 59 | metadata: 60 | name: fontbakery-dispatcher 61 | labels: 62 | component: fontbakery-dispatcher 63 | spec: 64 | ports: 65 | - port: 50051 66 | targetPort: 50051 67 | selector: 68 | component: fontbakery-dispatcher 69 | -------------------------------------------------------------------------------- /kubernetes/minikube-fontbakery-github-auth.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fontbakery-github-auth 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | component: fontbakery-github-auth 10 | template: 11 | metadata: 12 | labels: 13 | component: fontbakery-github-auth 14 | spec: 15 | containers: 16 | - name: fontbakery-github-auth 17 | image: fontbakery/base-javascript:1 18 | env: 19 | - name: FONTBAKERY_LOG_LEVEL 20 | value: "DEBUG" 21 | - name: GITHUB_OAUTH_CLIENT_ID 22 | valueFrom: 23 | secretKeyRef: 24 | name: external-resources 25 | key: github-oauth-client-id 26 | - name: GITHUB_OAUTH_CLIENT_SECRET 27 | valueFrom: 28 | secretKeyRef: 29 | name: external-resources 30 | key: github-oauth-client-secret 31 | - name: GITHUB_AUTH_ENGINEERS 32 | valueFrom: 33 | secretKeyRef: 34 | name: external-resources 35 | key: github-auth-engineers 36 | ports: 37 | - containerPort: 50051 38 | workingDir: /var/javascript 39 | command: ["node", "node/GitHubAuthServer.js", "-p", "50051"] 40 | --- 41 | apiVersion: v1 42 | kind: Service 43 | metadata: 44 | name: fontbakery-github-auth 45 | labels: 46 | component: fontbakery-github-auth 47 | spec: 48 | ports: 49 | - port: 50051 50 | targetPort: 50051 51 | selector: 52 | component: fontbakery-github-auth 53 | -------------------------------------------------------------------------------- /kubernetes/minikube-fontbakery-github-operations.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fontbakery-github-operations 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | component: fontbakery-github-operations 10 | template: 11 | metadata: 12 | labels: 13 | component: fontbakery-github-operations 14 | spec: 15 | containers: 16 | - name: fontbakery-github-operations 17 | image: fontbakery/base-javascript:1 18 | env: 19 | - name: FONTBAKERY_LOG_LEVEL 20 | value: "DEBUG" 21 | - name: GITHUB_API_TOKEN 22 | valueFrom: 23 | secretKeyRef: 24 | name: external-resources 25 | key: github-api-token 26 | ports: 27 | - containerPort: 50051 28 | workingDir: /var/javascript 29 | # command: ["node", "node/GitHubOperationsServer.js", "-p", "50051"] 30 | # WITH DEBUGGER 31 | command: ["node", "--inspect=0.0.0.0:9229", "node/GitHubOperationsServer.js", "-p", "50051"] 32 | # depends directly on rabbitmq/amqp 33 | # wait until rabbitmq can be reached before starting the actual container 34 | initContainers: 35 | - name: init-wait-for-rabbitmq 36 | image: busybox 37 | command: 38 | - sh 39 | - "-c" 40 | - > 41 | until wget -q -O - 42 | http://guest:guest@$RABBITMQ_MANAGEMENT_SERVICE_HOST:$RABBITMQ_MANAGEMENT_SERVICE_PORT/api/aliveness-test/%2F; 43 | do echo [`date`] waiting for rabbitmq: 10s; 44 | sleep 10; 45 | done; 46 | --- 47 | apiVersion: v1 48 | kind: Service 49 | metadata: 50 | name: fontbakery-github-operations 51 | labels: 52 | component: fontbakery-github-operations 53 | spec: 54 | ports: 55 | - port: 50051 56 | targetPort: 50051 57 | selector: 58 | component: fontbakery-github-operations 59 | -------------------------------------------------------------------------------- /kubernetes/minikube-fontbakery-init-workers.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fontbakery-init-workers 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | component: fontbakery-init-workers 10 | template: 11 | metadata: 12 | labels: 13 | component: fontbakery-init-workers 14 | role: init-workers 15 | spec: 16 | containers: 17 | - name: fontbakery-init-workers 18 | image: fontbakery/base-javascript:1 19 | env: 20 | - name: FONTBAKERY_LOG_LEVEL 21 | value: "DEBUG" 22 | - name: ENVIRONMENT_VERSION 23 | valueFrom: 24 | configMapKeyRef: 25 | name: env-config 26 | key: ENVIRONMENT_VERSION 27 | - name: RETHINKDB_PASSWORD 28 | valueFrom: 29 | secretKeyRef: 30 | name: external-resources 31 | key: rethinkdb-password 32 | ports: 33 | - containerPort: 50051 34 | workingDir: /var/javascript 35 | # command: ["node", "node/InitWorkers.js", "-p", "50051"] 36 | command: ["node", "--inspect=0.0.0.0:9222", "node/InitWorkers.js", "-p", "50051"] 37 | # depends directly on rabbitmq/amqp 38 | # wait until rabbitmq can be reached before starting the actual container 39 | initContainers: 40 | - name: init-wait-for-rabbitmq 41 | image: busybox 42 | command: 43 | - sh 44 | - "-c" 45 | - > 46 | until wget -q -O - 47 | http://guest:guest@$RABBITMQ_MANAGEMENT_SERVICE_HOST:$RABBITMQ_MANAGEMENT_SERVICE_PORT/api/aliveness-test/%2F; 48 | do echo [`date`] waiting for rabbitmq: 10s; 49 | sleep 10; 50 | done; 51 | --- 52 | apiVersion: v1 53 | kind: Service 54 | metadata: 55 | name: fontbakery-init-workers 56 | labels: 57 | component: fontbakery-init-workers 58 | spec: 59 | ports: 60 | - port: 50051 61 | targetPort: 50051 62 | selector: 63 | component: fontbakery-init-workers 64 | role: init-workers 65 | -------------------------------------------------------------------------------- /kubernetes/minikube-fontbakery-manifest-csvupstream.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fontbakery-manifest-csvupstream 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | component: fontbakery-manifest-csvupstream 10 | template: 11 | metadata: 12 | labels: 13 | component: fontbakery-manifest-csvupstream 14 | role: manifest 15 | spec: 16 | containers: 17 | - name: fontbakery-manifest-csvupstream 18 | image: fontbakery/base-javascript:1 19 | env: 20 | - name: FONTBAKERY_LOG_LEVEL 21 | value: "DEBUG" 22 | - name: DEVEL_FAMILY_WHITELIST 23 | value: "[\"ABeeZee\", \"Indie Flower\", \"Slabo\", \"Rosario\", \"Amiri\", \"Pacifico\", \"Astloch\"]" 24 | - name: CSV_SHEET_URL_SANDBOX 25 | value: https://docs.google.com/spreadsheets/d/1ODnp-yRYw1LrI3RTX-VZZsigPPieviE954sOsrlcx5o/pub?gid=0&single=true&output=csv 26 | - name: CSV_SHEET_URL_UPSTREAM 27 | value: https://docs.google.com/spreadsheets/d/1ampzD9veEdrwUMkOAJkMNkftqtv1jEygiPR0wZ6eNl8/pub?gid=0&single=true&output=csv 28 | workingDir: /var/javascript 29 | #command: ["node", "node/manifestSources/CSVSpreadsheet.js", "-p", "50051"] 30 | command: ["node", "--inspect=0.0.0.0:9229", "node/manifestSources/CSVSpreadsheet.js", "-p", "50051"] 31 | # depends directly on rabbitmq/amqp 32 | # wait until rabbitmq can be reached before starting the actual container 33 | initContainers: 34 | - name: init-wait-for-rabbitmq 35 | image: busybox 36 | command: 37 | - sh 38 | - "-c" 39 | - > 40 | until wget -q -O - 41 | http://guest:guest@$RABBITMQ_MANAGEMENT_SERVICE_HOST:$RABBITMQ_MANAGEMENT_SERVICE_PORT/api/aliveness-test/%2F; 42 | do echo [`date`] waiting for rabbitmq: 10s; 43 | sleep 10; 44 | done; 45 | --- 46 | apiVersion: v1 47 | kind: Service 48 | metadata: 49 | name: fontbakery-manifest-csvupstream 50 | labels: 51 | component: fontbakery-manifest-csvupstream 52 | spec: 53 | ports: 54 | - port: 50051 55 | targetPort: 50051 56 | selector: 57 | component: fontbakery-manifest-csvupstream 58 | role: manifest 59 | -------------------------------------------------------------------------------- /kubernetes/minikube-fontbakery-manifest-gfapi.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fontbakery-manifest-gfapi 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | component: fontbakery-manifest-gfapi 10 | template: 11 | metadata: 12 | labels: 13 | component: fontbakery-manifest-gfapi 14 | role: manifest 15 | spec: 16 | containers: 17 | - name: fontbakery-manifest-gfapi 18 | image: fontbakery/base-javascript:1 19 | env: 20 | - name: FONTBAKERY_LOG_LEVEL 21 | value: "DEBUG" 22 | - name: DEVEL_FAMILY_WHITELIST 23 | value: "[\"ABeeZee\", \"Indie Flower\", \"Slabo\", \"Rosario\", \"Amiri\", \"Pacifico\", \"Astloch\"]" 24 | - name: GOOGLE_API_KEY 25 | valueFrom: 26 | secretKeyRef: 27 | name: external-resources 28 | key: google-api-key 29 | workingDir: /var/javascript 30 | # command: ["node", "node/manifestSources/GoogleFonts.js", "-p", "50051"] 31 | command: ["node", "--inspect=0.0.0.0:9229", "node/manifestSources/GoogleFonts.js", "-p", "50051"] 32 | # depends directly on rabbitmq/amqp 33 | # wait until rabbitmq can be reached before starting the actual container 34 | initContainers: 35 | - name: init-wait-for-rabbitmq 36 | image: busybox 37 | command: 38 | - sh 39 | - "-c" 40 | - > 41 | until wget -q -O - 42 | http://guest:guest@$RABBITMQ_MANAGEMENT_SERVICE_HOST:$RABBITMQ_MANAGEMENT_SERVICE_PORT/api/aliveness-test/%2F; 43 | do echo [`date`] waiting for rabbitmq: 10s; 44 | sleep 10; 45 | done; 46 | --- 47 | apiVersion: v1 48 | kind: Service 49 | metadata: 50 | name: fontbakery-manifest-gfapi 51 | labels: 52 | component: fontbakery-manifest-gfapi 53 | spec: 54 | ports: 55 | - port: 50051 56 | targetPort: 50051 57 | selector: 58 | component: fontbakery-manifest-gfapi 59 | role: manifest 60 | -------------------------------------------------------------------------------- /kubernetes/minikube-fontbakery-manifest-githubgf.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fontbakery-manifest-githubgf 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | component: fontbakery-manifest-githubgf 10 | template: 11 | metadata: 12 | labels: 13 | component: fontbakery-manifest-githubgf 14 | role: manifest 15 | spec: 16 | containers: 17 | - name: fontbakery-manifest-githubgf 18 | image: fontbakery/base-javascript:1 19 | env: 20 | - name: FONTBAKERY_LOG_LEVEL 21 | value: "DEBUG" 22 | - name: DEVEL_FAMILY_WHITELIST 23 | value: "[\"ABeeZee\", \"Indie Flower\", \"Slabo\", \"Rosario\", \"Amiri\", \"Pacifico\", \"Astloch\"]" 24 | - name: GITHUB_API_TOKEN 25 | valueFrom: 26 | secretKeyRef: 27 | name: external-resources 28 | key: github-api-token 29 | workingDir: /var/javascript 30 | #command: ["node", "node/manifestSources/Git.js", "-p", "50051"] 31 | command: ["node", "--inspect=0.0.0.0:9229", "node/manifestSources/Git.js", "-p", "50051"] 32 | # depends directly on rabbitmq/amqp 33 | # wait until rabbitmq can be reached before starting the actual container 34 | initContainers: 35 | - name: init-wait-for-rabbitmq 36 | image: busybox 37 | command: 38 | - sh 39 | - "-c" 40 | - > 41 | until wget -q -O - 42 | http://guest:guest@$RABBITMQ_MANAGEMENT_SERVICE_HOST:$RABBITMQ_MANAGEMENT_SERVICE_PORT/api/aliveness-test/%2F; 43 | do echo [`date`] waiting for rabbitmq: 10s; 44 | sleep 10; 45 | done; 46 | --- 47 | apiVersion: v1 48 | kind: Service 49 | metadata: 50 | name: fontbakery-manifest-githubgf 51 | labels: 52 | component: fontbakery-manifest-githubgf 53 | spec: 54 | ports: 55 | - port: 50051 56 | targetPort: 50051 57 | selector: 58 | component: fontbakery-manifest-githubgf 59 | role: manifest 60 | -------------------------------------------------------------------------------- /kubernetes/minikube-fontbakery-manifest-main.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fontbakery-manifest-main 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | component: fontbakery-manifest-main 10 | template: 11 | metadata: 12 | labels: 13 | component: fontbakery-manifest-main 14 | role: manifest 15 | spec: 16 | containers: 17 | - name: fontbakery-manifest-main 18 | image: fontbakery/base-javascript:1 19 | env: 20 | - name: FONTBAKERY_LOG_LEVEL 21 | value: "DEBUG" 22 | - name: RETHINKDB_PASSWORD 23 | valueFrom: 24 | secretKeyRef: 25 | name: external-resources 26 | key: rethinkdb-password 27 | workingDir: /var/javascript 28 | command: ["node", "node/ManifestMain.js"] 29 | # depends directly on rabbitmq/amqp 30 | # wait until rabbitmq can be reached before starting the actual container 31 | initContainers: 32 | - name: init-wait-for-rabbitmq 33 | image: busybox 34 | command: 35 | - sh 36 | - "-c" 37 | - > 38 | until wget -q -O - 39 | http://guest:guest@$RABBITMQ_MANAGEMENT_SERVICE_HOST:$RABBITMQ_MANAGEMENT_SERVICE_PORT/api/aliveness-test/%2F; 40 | do echo [`date`] waiting for rabbitmq: 10s; 41 | sleep 10; 42 | done; 43 | -------------------------------------------------------------------------------- /kubernetes/minikube-fontbakery-reports.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fontbakery-reports 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | component: fontbakery-reports 10 | template: 11 | metadata: 12 | labels: 13 | component: fontbakery-reports 14 | role: reportsserver 15 | spec: 16 | containers: 17 | - name: fontbakery-reports 18 | image: fontbakery/base-javascript:1 19 | env: 20 | - name: FONTBAKERY_LOG_LEVEL 21 | value: "DEBUG" 22 | - name: RETHINKDB_PASSWORD 23 | valueFrom: 24 | secretKeyRef: 25 | name: external-resources 26 | key: rethinkdb-password 27 | ports: 28 | - containerPort: 50051 29 | workingDir: /var/javascript 30 | # command: ["node", "node/ReportsServer.js", "-p", "50051"] 31 | command: ["node", "--inspect=0.0.0.0:9229", "node/ReportsServer.js", "-p", "50051"] 32 | --- 33 | apiVersion: v1 34 | kind: Service 35 | metadata: 36 | name: fontbakery-reports 37 | labels: 38 | component: fontbakery-reports 39 | spec: 40 | ports: 41 | - port: 50051 42 | targetPort: 50051 43 | selector: 44 | component: fontbakery-reports 45 | role: reportsserver 46 | -------------------------------------------------------------------------------- /kubernetes/minikube-fontbakery-storage-cache.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fontbakery-storage-cache 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | component: fontbakery-storage-cache 10 | template: 11 | metadata: 12 | labels: 13 | component: fontbakery-storage-cache 14 | role: messagecache 15 | spec: 16 | containers: 17 | - name: fontbakery-storage-cache 18 | image: fontbakery/base-javascript:1 19 | env: 20 | - name: FONTBAKERY_LOG_LEVEL 21 | value: "DEBUG" 22 | ports: 23 | - containerPort: 50051 24 | workingDir: /var/javascript 25 | command: ["node", "node/CacheServer.js", "-p", "50051"] 26 | --- 27 | apiVersion: v1 28 | kind: Service 29 | metadata: 30 | name: fontbakery-storage-cache 31 | labels: 32 | component: fontbakery-storage-cache 33 | spec: 34 | ports: 35 | - port: 50051 36 | targetPort: 50051 37 | selector: 38 | component: fontbakery-storage-cache 39 | role: messagecache 40 | -------------------------------------------------------------------------------- /kubernetes/minikube-fontbakery-storage-persistence.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fontbakery-storage-persistence 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | component: fontbakery-storage-persistence 10 | template: 11 | metadata: 12 | labels: 13 | component: fontbakery-storage-persistence 14 | role: messagecache 15 | spec: 16 | containers: 17 | - name: fontbakery-storage-persistence 18 | image: fontbakery/base-javascript:1 19 | env: 20 | - name: FONTBAKERY_LOG_LEVEL 21 | value: "DEBUG" 22 | ports: 23 | - containerPort: 50051 24 | workingDir: /var/javascript 25 | command: ["node", "node/PersistenceServer.js", "-p", "50051"] 26 | --- 27 | apiVersion: v1 28 | kind: Service 29 | metadata: 30 | name: fontbakery-storage-persistence 31 | labels: 32 | component: fontbakery-storage-persistence 33 | spec: 34 | ports: 35 | - port: 50051 36 | targetPort: 50051 37 | selector: 38 | component: fontbakery-storage-persistence 39 | role: messagecache 40 | -------------------------------------------------------------------------------- /kubernetes/minikube-fontbakery-worker.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: fontbakery-worker 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | run: fontbakery-worker 10 | template: 11 | metadata: 12 | labels: 13 | run: fontbakery-worker 14 | spec: 15 | containers: 16 | - name: fontbakery-worker 17 | image: fontbakery/base-python:1 18 | env: 19 | - name: FONTBAKERY_WORKER_LOG_LEVEL 20 | value: "DEBUG" 21 | - name: BROWSERSTACK_USERNAME 22 | valueFrom: 23 | secretKeyRef: 24 | name: external-resources 25 | key: browserstack-username 26 | - name: BROWSERSTACK_ACCESS_KEY 27 | valueFrom: 28 | secretKeyRef: 29 | name: external-resources 30 | key: browserstack-access-key 31 | - name: RETHINKDB_PASSWORD 32 | valueFrom: 33 | secretKeyRef: 34 | name: external-resources 35 | key: rethinkdb-password 36 | workingDir: /var/python 37 | command: ["python3", "-u", "worker-launcher.py"] 38 | # depends directly on rabbitmq/amqp 39 | # wait until rabbitmq can be reached before starting the actual container 40 | initContainers: 41 | - name: init-wait-for-rabbitmq 42 | image: busybox 43 | command: 44 | - sh 45 | - "-c" 46 | - > 47 | until wget -q -O - 48 | http://guest:guest@$RABBITMQ_MANAGEMENT_SERVICE_HOST:$RABBITMQ_MANAGEMENT_SERVICE_PORT/api/aliveness-test/%2F; 49 | do echo [`date`] waiting for rabbitmq: 10s; 50 | sleep 10; 51 | done; 52 | -------------------------------------------------------------------------------- /kubernetes/minikube-rabbitmq.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | labels: 5 | component: rabbitmq 6 | name: rabbitmq 7 | namespace: fontbakery 8 | spec: 9 | serviceName: rabbitmq 10 | replicas: 1 11 | selector: 12 | matchLabels: 13 | component: rabbitmq 14 | template: 15 | metadata: 16 | labels: 17 | app: taskQueue 18 | component: rabbitmq 19 | spec: 20 | containers: 21 | - image: rabbitmq:3-management-alpine 22 | name: rabbitmq 23 | ports: 24 | - name: http 25 | protocol: TCP 26 | containerPort: 15672 27 | - name: amqp 28 | protocol: TCP 29 | containerPort: 5672 30 | - name: epmd 31 | containerPort: 4369 32 | resources: 33 | limits: 34 | cpu: 100m 35 | imagePullPolicy: Always 36 | #volumes: 37 | # - name: config-volume 38 | # configMap: 39 | # name: rabbitmq-config 40 | # items: 41 | # - key: rabbitmq.conf 42 | # path: rabbitmq.conf 43 | # - key: enabled_plugins 44 | # path: enabled_plugins 45 | --- 46 | apiVersion: v1 47 | kind: Service 48 | metadata: 49 | name: rabbitmq-management 50 | labels: 51 | component: rabbitmq 52 | spec: 53 | type: LoadBalancer 54 | ports: 55 | - port: 8888 56 | targetPort: 15672 57 | name: http 58 | selector: 59 | component: rabbitmq 60 | --- 61 | apiVersion: v1 62 | kind: Service 63 | metadata: 64 | name: rabbitmq-service 65 | labels: 66 | component: rabbitmq 67 | spec: 68 | ports: 69 | - port: 5672 70 | selector: 71 | app: taskQueue 72 | component: rabbitmq 73 | -------------------------------------------------------------------------------- /kubernetes/minikube-rethinkdb.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | labels: 5 | db: rethinkdb 6 | name: rethinkdb-replica-1 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | db: rethinkdb 12 | template: 13 | metadata: 14 | labels: 15 | db: rethinkdb 16 | role: replica 17 | instance: one 18 | spec: 19 | containers: 20 | - image: fontbakery/rethinkdb:2.3.6-fontbakery-1 21 | args: 22 | - "/run.sh" 23 | - "--bind" 24 | - "all" 25 | - "--cache-size" 26 | - "100" 27 | imagePullPolicy: IfNotPresent 28 | name: rethinkdb 29 | env: 30 | - name: RETHINKDB_PASSWORD 31 | valueFrom: 32 | secretKeyRef: 33 | name: external-resources 34 | key: rethinkdb-password 35 | - name: RETHINK_CLUSTER_SERVICE 36 | value: rethinkdb 37 | - name: POD_IP 38 | valueFrom: 39 | fieldRef: 40 | apiVersion: v1 41 | fieldPath: status.podIP 42 | - name: POD_NAMESPACE 43 | valueFrom: 44 | fieldRef: 45 | fieldPath: metadata.namespace 46 | - name: POD_NAME 47 | valueFrom: 48 | fieldRef: 49 | fieldPath: metadata.name 50 | - name: POD_IP 51 | valueFrom: 52 | fieldRef: 53 | apiVersion: v1 54 | fieldPath: status.podIP 55 | - name: RETHINK_CLUSTER 56 | value: rethinkdb 57 | ports: 58 | - containerPort: 8080 59 | name: admin 60 | - containerPort: 28015 61 | name: driver 62 | - containerPort: 29015 63 | name: cluster 64 | livenessProbe: 65 | exec: 66 | command: 67 | - /rethinkdb-probe 68 | failureThreshold: 3 69 | initialDelaySeconds: 15 70 | periodSeconds: 10 71 | successThreshold: 1 72 | timeoutSeconds: 5 73 | readinessProbe: 74 | exec: 75 | command: 76 | - /rethinkdb-probe 77 | failureThreshold: 3 78 | initialDelaySeconds: 15 79 | periodSeconds: 10 80 | successThreshold: 1 81 | timeoutSeconds: 5 82 | resources: 83 | limits: 84 | cpu: 100m 85 | memory: 512Mi 86 | requests: 87 | cpu: 100m 88 | memory: 512Mi 89 | volumeMounts: 90 | - mountPath: /data 91 | name: storage 92 | volumes: 93 | - name: storage 94 | emptyDir: {} 95 | --- 96 | apiVersion: v1 97 | kind: Service 98 | metadata: 99 | labels: 100 | db: rethinkdb 101 | name: rethinkdb-admin 102 | spec: 103 | type: LoadBalancer 104 | ports: 105 | - port: 8080 106 | targetPort: 8080 107 | selector: 108 | db: rethinkdb 109 | role: replica 110 | --- 111 | apiVersion: v1 112 | kind: Service 113 | metadata: 114 | labels: 115 | db: rethinkdb 116 | name: rethinkdb-driver 117 | spec: 118 | ports: 119 | - port: 28015 120 | targetPort: 28015 121 | selector: 122 | db: rethinkdb 123 | role: replica 124 | -------------------------------------------------------------------------------- /skaffold.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: skaffold/v1 2 | kind: Config 3 | metadata: 4 | name: fontbakery-dashboard 5 | build: 6 | artifacts: 7 | - image: fontbakery/base-javascript 8 | context: containers/base/javascript 9 | sync: 10 | infer: 11 | - 'browser/**' 12 | - image: fontbakery/base-python 13 | context: containers/base/python 14 | - image: fontbakery/rethinkdb 15 | context: containers/rethinkdb 16 | deploy: 17 | kubectl: 18 | manifests: 19 | - kubernetes/minikube-*.yaml 20 | portForward: 21 | - resourceType: service 22 | resourceName: fontbakery-api 23 | namespace: fontbakery 24 | port: 80 25 | localPort: 3000 26 | --------------------------------------------------------------------------------