├── .editorconfig ├── .firebaserc ├── .github ├── FUNDING.yml ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── feature_request.md │ └── other.md ├── pull_request_template.md └── workflows │ └── main.yml ├── .gitignore ├── .nvmrc ├── .prettierignore ├── .prettierrc.json ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── DEVELOPMENT.md ├── LICENSE ├── README.md ├── firebase.json ├── firestore.indexes.json ├── firestore.rules ├── functions ├── .eslintrc.js ├── .gitignore ├── .yarnrc.yml ├── package.json ├── src │ ├── api │ │ ├── index.ts │ │ ├── queueStatus.ts │ │ ├── repoVersions.ts │ │ ├── reportBuildFailure.ts │ │ ├── reportNewBuild.ts │ │ ├── reportPublication.ts │ │ ├── retryBuild.ts │ │ ├── testFunction.ts │ │ └── unityVersions.ts │ ├── config │ │ ├── credential.ts │ │ ├── settings.ts │ │ └── token.ts │ ├── cron │ │ └── index.ts │ ├── index.ts │ ├── logic │ │ ├── buildQueue │ │ │ ├── cleanUpBuilds.ts │ │ │ ├── cleaner.ts │ │ │ ├── index.ts │ │ │ ├── ingeminator.ts │ │ │ ├── scheduleBuildsFromTheQueue.ts │ │ │ └── scheduler.ts │ │ ├── dataTransformation │ │ │ ├── dataMigrations.ts │ │ │ └── index.ts │ │ ├── ingestRepoVersions │ │ │ ├── index.ts │ │ │ ├── scrapeVersions.ts │ │ │ └── updateDatabase.ts │ │ ├── ingestUnityVersions │ │ │ ├── index.ts │ │ │ ├── scrapeVersions.ts │ │ │ └── updateDatabase.ts │ │ └── init │ │ │ └── assignDefaultAdmins.ts │ ├── model-triggers │ │ ├── editorVersionInfo.ts │ │ ├── index.ts │ │ └── repoVersionInfo.ts │ ├── model │ │ ├── ciBuilds.ts │ │ ├── ciJobs.ts │ │ ├── ciVersionInfo.ts │ │ ├── editorVersionInfo.ts │ │ ├── gitHubWorkflow.ts │ │ ├── image.ts │ │ └── repoVersionInfo.ts │ └── service │ │ ├── discord.ts │ │ ├── dockerhub.ts │ │ ├── firebase.ts │ │ └── github.ts ├── tsconfig.json └── yarn.lock ├── package.json ├── public ├── 404.html └── index.html ├── storage.rules └── yarn.lock /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | indent_size = 2 7 | indent_style = space 8 | insert_final_newline = true 9 | max_line_length = 100 10 | tab_width = 2 11 | trim_trailing_whitespace = true 12 | 13 | [*.md] 14 | max_line_length = off 15 | trim_trailing_whitespace = false 16 | 17 | [*.{yml,yaml}] 18 | max_line_length = off 19 | 20 | [COMMIT_EDITMSG] 21 | max_line_length = off 22 | -------------------------------------------------------------------------------- /.firebaserc: -------------------------------------------------------------------------------- 1 | { 2 | "projects": { 3 | "default": "unity-ci-versions" 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: game-ci 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # replace with a single OpenCollective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 13 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | --- 8 | 9 | **Describe the bug** 10 | A clear and concise description of what the bug is. 11 | 12 | **To Reproduce** 13 | Steps to reproduce the behavior: 14 | 15 | - 16 | 17 | **Expected behavior** 18 | A clear and concise description of what you expected to happen. 19 | 20 | **Screenshots** 21 | If applicable, add screenshots to help explain your problem. 22 | 23 | **Additional context** 24 | Add any other context about the problem here. 25 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | --- 8 | 9 | **Is your feature request related to a problem? Please describe.** 10 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 11 | 12 | **Describe the solution you'd like** 13 | A clear and concise description of what you want to happen. 14 | 15 | **Describe alternatives you've considered** 16 | A clear and concise description of any alternative solutions or features you've considered. 17 | 18 | **Additional context** 19 | Add any other context or screenshots about the feature request here. 20 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/other.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Other 3 | about: Something else 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | --- 8 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | #### Changes 2 | 3 | - ... 4 | 5 | #### Checklist 6 | 7 | - [x] Read the contribution [guide](../CONTRIBUTING.md) and accept the [code](../CODE_OF_CONDUCT.md) of conduct 8 | - [ ] Readme (updated or not needed) 9 | - [ ] Tests (added, updated or not needed) 10 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: 🚀 2 | 3 | on: 4 | pull_request: {} 5 | push: { branches: [main] } 6 | 7 | jobs: 8 | test: 9 | name: 🧪 Test 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v4 13 | - uses: actions/setup-node@v4 14 | with: 15 | node-version: 20 16 | cache: 'yarn' 17 | - name: install dependencies 18 | run: yarn && yarn --cwd ./functions 19 | - name: run linter 20 | run: yarn lint && yarn --cwd ./functions lint 21 | - name: run tests 22 | run: yarn test && yarn --cwd ./functions test 23 | # - name: Upload test results 24 | # uses: actions/upload-artifact@v1 25 | # with: 26 | # name: Test results 27 | # path: "**/artifacs" 28 | 29 | build: 30 | name: 🛠 Build 31 | runs-on: ubuntu-latest 32 | steps: 33 | - uses: actions/checkout@v4 34 | - uses: actions/setup-node@v4 35 | with: 36 | node-version: 20 37 | cache: 'yarn' 38 | - name: install dependencies 39 | run: yarn && yarn --cwd ./functions 40 | - name: build 41 | run: yarn --cwd ./functions build 42 | 43 | testDeploy: 44 | name: Test Deploy 45 | needs: [test, build] 46 | runs-on: ubuntu-latest 47 | steps: 48 | - uses: actions/checkout@v4 49 | - uses: actions/setup-node@v4 50 | with: 51 | node-version: 20 52 | cache: 'yarn' 53 | - name: install dependencies 54 | run: yarn && yarn --cwd ./functions 55 | - name: Deploy test to Firebase 56 | uses: w9jds/firebase-action@v13.25.0 57 | with: 58 | args: deploy --only functions:testFunction 59 | env: 60 | GCP_SA_KEY: '${{ secrets.FIREBASE_SERVICE_ACCOUNT_UNITY_CI_VERSIONS }}' 61 | 62 | - name: Call Test Function 63 | run: curl -f -s -S -X POST https://testfunction-wbe4ukn6tq-ey.a.run.app 64 | 65 | - name: Cleanup Firebase Test 66 | uses: w9jds/firebase-action@v13.25.0 67 | if: always() 68 | with: 69 | args: functions:delete testFunction --force 70 | env: 71 | GCP_SA_KEY: '${{ secrets.FIREBASE_SERVICE_ACCOUNT_UNITY_CI_VERSIONS }}' 72 | 73 | deploy: 74 | name: ✨ Deploy 75 | needs: [test, build, testDeploy] 76 | runs-on: ubuntu-latest 77 | if: github.event_name == 'push' && github.ref == 'refs/heads/main' 78 | steps: 79 | - uses: actions/checkout@v4 80 | - uses: actions/setup-node@v4 81 | with: 82 | node-version: 20 83 | cache: 'yarn' 84 | - name: install dependencies 85 | run: yarn && yarn --cwd ./functions 86 | - name: Deploy to Firebase 87 | uses: w9jds/firebase-action@v13.25.0 88 | with: 89 | args: deploy 90 | env: 91 | GCP_SA_KEY: '${{ secrets.FIREBASE_SERVICE_ACCOUNT_UNITY_CI_VERSIONS }}' 92 | - name: Cleanup Firebase Test 93 | uses: w9jds/firebase-action@v13.25.0 94 | if: always() 95 | with: 96 | args: functions:delete testFunction --force 97 | env: 98 | GCP_SA_KEY: '${{ secrets.FIREBASE_SERVICE_ACCOUNT_UNITY_CI_VERSIONS }}' 99 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | firebase-debug.log* 8 | firebase-debug.*.log* 9 | 10 | # Firebase cache 11 | .firebase/ 12 | 13 | # Firebase config 14 | 15 | # Uncomment this if you'd like others to create their own Firebase project. 16 | # For a team working on the same Firebase project(s), it is recommended to leave 17 | # it commented so all members can deploy to the same project(s) in .firebaserc. 18 | # .firebaserc 19 | 20 | # Runtime data 21 | pids 22 | *.pid 23 | *.seed 24 | *.pid.lock 25 | 26 | # Directory for instrumented libs generated by jscoverage/JSCover 27 | lib-cov 28 | 29 | # Coverage directory used by tools like istanbul 30 | coverage 31 | 32 | # nyc test coverage 33 | .nyc_output 34 | 35 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 36 | .grunt 37 | 38 | # Bower dependency directory (https://bower.io/) 39 | bower_components 40 | 41 | # node-waf configuration 42 | .lock-wscript 43 | 44 | # Compiled binary addons (http://nodejs.org/api/addons.html) 45 | build/Release 46 | 47 | # Dependency directories 48 | node_modules/ 49 | 50 | # Optional npm cache directory 51 | .npm 52 | 53 | # Optional eslint cache 54 | .eslintcache 55 | 56 | # Optional REPL history 57 | .node_repl_history 58 | 59 | # Output of 'npm pack' 60 | *.tgz 61 | 62 | # Yarn Integrity file 63 | .yarn-integrity 64 | 65 | # dotenv environment variables file 66 | .env 67 | 68 | # IDE 69 | .idea 70 | .vs 71 | .vscode 72 | functions/.yarn/ 73 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | v10.22.1 2 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | **/node_modules/** 2 | **/public/** 3 | **/lib/** 4 | -------------------------------------------------------------------------------- /.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "semi": true, 3 | "singleQuote": true, 4 | "trailingComma": "all", 5 | "printWidth": 100 6 | } 7 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | - Using welcoming and inclusive language 18 | - Being respectful of differing viewpoints and experiences 19 | - Gracefully accepting constructive criticism 20 | - Focusing on what is best for the community 21 | - Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | - The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | - Trolling, insulting/derogatory comments, and personal or political attacks 28 | - Public or private harassment 29 | - Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | - Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at webber@takken.io. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | ## How to Contribute 4 | 5 | #### Code of Conduct 6 | 7 | This repository has adopted the Contributor Covenant as it's 8 | Code of Conduct. It is expected that participants adhere to it. 9 | 10 | #### Proposing a Change 11 | 12 | If you are unsure about whether or not a change is desired, 13 | you can create an issue. This is useful because it creates 14 | the possibility for a discussion that's visible to everyone. 15 | 16 | When fixing a bug it is fine to submit a pull request right away. 17 | 18 | #### Sending a Pull Request 19 | 20 | Steps to be performed to submit a pull request: 21 | 22 | 1. Fork the repository and create your branch from `main`. 23 | 2. Run `yarn` in the repository root. 24 | 3. If you've fixed a bug or added code that should be tested, add tests! 25 | 4. Fill out the description, link any related issues and submit your pull request. 26 | 27 | #### Pull Request Prerequisites 28 | 29 | ##### Tools 30 | 31 | You need the following tools to be installed. 32 | 33 | - [Node](https://nodejs.org/) installed at v10.X. (because cloud functions) 34 | - [npm](https://www.npmjs.com/) latest (because of firebase) 35 | - [firebase](https://firebase.google.com/docs/cli) latest 36 | 37 | > **Tip:** _Use 38 | > [nvm](https://github.com/nvm-sh/nvm) or 39 | > [n](https://github.com/tj/n) or 40 | > [nodenv](https://github.com/nodenv/nodenv) 41 | > to manage Node.js versions on your machine._ 42 | 43 | ##### Plugins 44 | 45 | Install and enable plugins for your IDE: 46 | 47 | - ESLint 48 | - Prettier - _Enable auto format on save: 49 | ([WebStorm](https://www.jetbrains.com/help/idea/prettier.html#ws_prettier_configure), 50 | [PhpStorm](https://www.jetbrains.com/help/idea/prettier.html#ws_prettier_configure), 51 | [VS Code](https://marketplace.visualstudio.com/items?itemName=esbenp.prettier-vscode#format-on-save))._ 52 | 53 | ##### Knowledge 54 | 55 | Please note that commit hooks will run automatically to perform some tasks; 56 | 57 | - format your code 58 | - run tests 59 | - build distributable files 60 | 61 | #### License 62 | 63 | By contributing to this repository, you agree that your contributions will be licensed under its MIT license. 64 | -------------------------------------------------------------------------------- /DEVELOPMENT.md: -------------------------------------------------------------------------------- 1 | # Development Guide 2 | 3 | This guide helps you set up the development environment for the GameCI Versioning Backend. 4 | 5 | ## Prerequisites 6 | 7 | - [Node.js](https://nodejs.org/) v20+ (required for Firebase Functions) 8 | - [Yarn](https://yarnpkg.com/) for package management 9 | - [Firebase CLI](https://firebase.google.com/docs/cli) for local development and deployment 10 | - [Java Runtime Environment](https://www.java.com/) for Firebase emulators 11 | 12 | > **Tip:** Use [nvm](https://github.com/nvm-sh/nvm), [n](https://github.com/tj/n), or [volta](https://volta.sh/) to manage Node.js versions. 13 | 14 | ## Setup 15 | 16 | ### 1. Install Firebase CLI 17 | 18 | ```bash 19 | npm i -g firebase-tools 20 | ``` 21 | 22 | ### 2. Install Dependencies 23 | 24 | Install dependencies in both root repository and `functions` directory: 25 | 26 | ```bash 27 | yarn install 28 | ``` 29 | 30 | ```bash 31 | cd functions 32 | yarn install 33 | ``` 34 | 35 | ### 3. Build Functions Code 36 | 37 | Before running the emulators, you need to build the functions code: 38 | 39 | ```bash 40 | cd functions 41 | yarn build 42 | ``` 43 | 44 | This will compile the TypeScript code into JavaScript in the `lib` directory, which the emulator needs to run the functions. 45 | 46 | ### 4. Set Up Credentials 47 | 48 | > **Note**: For basic local development and testing, you can skip this step initially. You'll see some warnings in the emulator output, but most functionality will still work. If you need to test integrations or need full functionality, follow these steps. 49 | 50 | #### Firebase Admin SDK 51 | 52 | To use Firebase Admin SDK locally: 53 | 54 | 1. Download a service account key from your Firebase project settings: 55 | - Go to [Firebase Console](https://console.firebase.google.com/) 56 | - Select your project 57 | - Go to Project Settings > Service accounts 58 | - Click "Generate new private key" 59 | - Save the JSON file securely 60 | 61 | 2. Set the environment variable to the key file: 62 | 63 | **Linux / macOS**: 64 | ```bash 65 | export GOOGLE_APPLICATION_CREDENTIALS="/path/to/serviceAccountKey.json" 66 | ``` 67 | 68 | **Windows (PowerShell)**: 69 | ```powershell 70 | $env:GOOGLE_APPLICATION_CREDENTIALS="C:\path\to\serviceAccountKey.json" 71 | ``` 72 | 73 | > Note: Without this setup, you'll see warnings about being unable to fetch Admin SDK configuration, but the emulators will still run in a limited capacity. 74 | 75 | #### Integration Environment Variables 76 | 77 | To test integrations locally, set these environment variables: 78 | 79 | **Discord**: 80 | ```bash 81 | export DISCORD_TOKEN="your_discord_token" 82 | ``` 83 | 84 | **GitHub**: 85 | ```bash 86 | export GITHUB_CLIENT_SECRET="your_github_app_client_secret" 87 | export GITHUB_PRIVATE_KEY="your_github_app_private_key" 88 | ``` 89 | 90 | **Internal Token**: 91 | ```bash 92 | export INTERNAL_TOKEN="your_internal_token" 93 | ``` 94 | 95 | > The internal token is used for self-authentication and communication with the [docker repo](https://github.com/Unity-CI/docker). 96 | 97 | ### 5. Run Locally 98 | 99 | ```bash 100 | firebase emulators:start 101 | ``` 102 | 103 | This starts the Firebase emulators for Functions and Firestore, with hosting on port 5002 to avoid common port conflicts. 104 | 105 | #### Prerequisites for Emulators 106 | 107 | - **Java Runtime Environment**: Firebase emulators require Java to be installed and available on your system PATH 108 | - **Firebase Login**: Run `firebase login` to authenticate the CLI 109 | 110 | #### Port Configuration 111 | 112 | This project's `firebase.json` is already configured to use port 5002 for the hosting emulator instead of the default port 5000, which helps avoid conflicts with AirPlay Receiver on macOS. 113 | 114 | You're free to customize these ports for your local development environment as needed. If you encounter port conflicts, you can modify the port numbers in your `firebase.json`: 115 | 116 | ```json 117 | { 118 | "emulators": { 119 | "hosting": { 120 | "port": 5002 121 | }, 122 | "functions": { 123 | "port": 5001 124 | }, 125 | "firestore": { 126 | "port": 8080 127 | } 128 | } 129 | } 130 | ``` 131 | 132 | Modifying these ports only affects local development environment and won't impact deployment. 133 | 134 | ## Credentials Setup 135 | 136 | ### Firebase Admin SDK 137 | 138 | To use Firebase Admin SDK locally: 139 | 140 | 1. Download a service account key from your Firebase project settings 141 | 2. Set the environment variable to the key file 142 | 143 | **Linux / macOS**: 144 | ```bash 145 | export GOOGLE_APPLICATION_CREDENTIALS="/path/to/serviceAccountKey.json" 146 | ``` 147 | 148 | **Windows (PowerShell)**: 149 | ```powershell 150 | $env:GOOGLE_APPLICATION_CREDENTIALS="C:\path\to\serviceAccountKey.json" 151 | ``` 152 | 153 | ### Integration Environment Variables 154 | 155 | To test integrations locally, set these environment variables: 156 | 157 | **Discord**: 158 | ```bash 159 | export DISCORD_TOKEN="your_discord_token" 160 | ``` 161 | 162 | **GitHub**: 163 | ```bash 164 | export GITHUB_CLIENT_SECRET="your_github_app_client_secret" 165 | export GITHUB_PRIVATE_KEY="your_github_app_private_key" 166 | ``` 167 | 168 | **Internal Token**: 169 | ```bash 170 | export INTERNAL_TOKEN="your_internal_token" 171 | ``` 172 | 173 | > The internal token is used for self-authentication and communication with the [docker repo](https://github.com/Unity-CI/docker). 174 | 175 | ## Deployment 176 | 177 | > **Note:** You need project access to deploy. 178 | 179 | 1. Login to Firebase: 180 | ```bash 181 | firebase login 182 | ``` 183 | 184 | 2. Deploy everything: 185 | ```bash 186 | firebase deploy 187 | ``` 188 | 189 | Or deploy specific services: 190 | ```bash 191 | firebase deploy --only functions 192 | ``` 193 | 194 | ## Firebase Configuration 195 | 196 | Update environment variables in Firebase (useful when rotating tokens or migrating environments): 197 | 198 | ```bash 199 | firebase functions:config:set discord.token="your_discord_token" 200 | firebase functions:config:set github.client-secret="your_github_app_client_secret" 201 | firebase functions:config:set github.private-key="your_github_app_private_key" 202 | firebase functions:config:set internal.token="your_internal_token" 203 | ``` 204 | 205 | > Note: Firebase Functions configuration uses dot notation (e.g., `internal.token`) when setting with the CLI, but when using environment variables locally, use uppercase without dots (e.g., `INTERNAL_TOKEN`). This is due to how Firebase handles different configuration methods. 206 | 207 | ## Development Workflow 208 | 209 | 1. Make changes to code in `functions/src/` 210 | 2. Build the functions code: 211 | ```bash 212 | cd functions && yarn build 213 | ``` 214 | 3. Run the emulator to test locally: 215 | ```bash 216 | firebase emulators:start 217 | ``` 218 | 4. For hot reloading during development: 219 | ```bash 220 | # In one terminal 221 | cd functions && yarn watch 222 | 223 | # In another terminal 224 | firebase emulators:start 225 | ``` 226 | 5. Test your changes 227 | 6. Deploy when ready 228 | 229 | ## Troubleshooting 230 | 231 | - **Firebase Login Issues**: Make sure you have access to the Firebase project 232 | - **Emulator Port Conflicts**: 233 | - Check for services using ports 4000, 5001, 8080, or 9000 234 | - This project uses port 5002 for hosting to avoid conflicts with AirPlay Receiver on macOS 235 | - Feel free to change any port in your local `firebase.json` if you encounter conflicts 236 | - See the [Port Configuration](#port-configuration) section for details 237 | - **Java Not Found Error**: 238 | - The Firebase emulators require Java to be installed 239 | - On macOS, install Java using `brew install openjdk@17` or download from [java.com](https://www.java.com) 240 | - Make sure Java is on your PATH: `java -version` should return the installed version 241 | - **Missing Functions Library Error** (`functions/lib/index.js does not exist`): 242 | - This indicates that the TypeScript code hasn't been compiled 243 | - Run `cd functions && yarn build` to compile the code 244 | - If that doesn't work, check for TypeScript compilation errors in the build output 245 | - **Admin SDK Configuration Errors**: 246 | - Set up the `GOOGLE_APPLICATION_CREDENTIALS` environment variable as described in the [Set Up Credentials](#4-set-up-credentials) section 247 | - For testing, you can often ignore this warning as the emulators will still run with limited functionality 248 | - **Integration Issues**: 249 | - Ensure all required environment variables are correctly set 250 | - For local development without integration testing, you can often proceed without setting these variables 251 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020-present Unity CI 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GameCI Versioning Backend 2 | 3 | The GameCI Versioning Backend automates the tracking, scheduling, and building of Unity versions and Docker images for the GameCI ecosystem. It connects with GitHub Actions for CI/CD workflows and Discord for notifications. 4 | 5 | ## System Overview 6 | 7 | ```mermaid 8 | graph TD 9 | A[Unity Version Archive] -->|Scrape & Detect| B[Version Ingest] 10 | B -->|Store| C[Firestore Database] 11 | B -->|Notify| D[Discord] 12 | C -->|Schedule| E[CI Job Scheduler] 13 | E -->|Trigger| F[GitHub Actions Workflows] 14 | F -->|Report Status| G[CI Build Reporters] 15 | G -->|Update| C 16 | H[Ingeminator] -->|Retry Failed Builds| F 17 | C -->|Monitor| H 18 | ``` 19 | 20 | ## Unity Version Ingest 21 | 22 | The backend regularly scrapes Unity version information: 23 | 24 | 1. Uses the [`unity-changeset` package](https://github.com/mob-sakai/unity-changeset) from [mob-sakai](https://github.com/mob-sakai) to detect new Unity versions 25 | 2. Filters versions (only stable versions 2017+) 26 | 3. Stores version details in Firestore 27 | 4. Notifies maintainers via Discord 28 | 5. Schedules build jobs for new versions 29 | 30 | ## CI Job Workflow 31 | 32 | Each Unity version generates CI jobs and builds with the following relationships: 33 | 34 | ``` 35 | CiJob (e.g., Unity 2022.3.15f1) 36 | ├── CiBuild: ubuntu-2022.3.15f1-webgl 37 | ├── CiBuild: ubuntu-2022.3.15f1-android 38 | ├── CiBuild: windows-2022.3.15f1-webgl 39 | └── ... (other baseOS-version-targetPlatform combinations) 40 | ``` 41 | 42 | ## Scheduler 43 | 44 | The scheduler coordinates building Docker images: 45 | 46 | - First ensures base and hub images are built 47 | - Monitors for failed jobs and triggers the Ingeminator to retry them 48 | - Prioritizes jobs based on Unity version recency 49 | - Limits concurrent jobs to prevent overloading GitHub Actions 50 | 51 | ## Ingeminator 52 | 53 | The Ingeminator ("repeater") handles the reliability of the build system: 54 | 55 | - Detects failed builds and reschedules them 56 | - Implements an exponential backoff strategy for retries 57 | - Alerts via Discord when builds reach maximum retry attempts 58 | - Works with the scheduler to manage retry priorities 59 | 60 | ## Database Backup 61 | 62 | Back up the Firestore database: 63 | ```bash 64 | export GOOGLE_APPLICATION_CREDENTIALS="/path/to/serviceAccountKey.json" 65 | yarn run backfire export ./export/versioningBackendBackup --project unity-ci-versions --keyFile $GOOGLE_APPLICATION_CREDENTIALS 66 | ``` 67 | 68 | Restore a backup: 69 | ```bash 70 | yarn run backfire import ./export/versioningBackendBackup --project unity-ci-versions --keyFile $GOOGLE_APPLICATION_CREDENTIALS 71 | ``` 72 | 73 | ## Development 74 | 75 | For instructions on setting up the development environment, see [DEVELOPMENT.md](./DEVELOPMENT.md). 76 | 77 | ## Contributing 78 | 79 | We welcome contributions! See [CONTRIBUTING.md](./CONTRIBUTING.md) for guidelines. 80 | -------------------------------------------------------------------------------- /firebase.json: -------------------------------------------------------------------------------- 1 | { 2 | "firestore": { 3 | "rules": "firestore.rules", 4 | "indexes": "firestore.indexes.json" 5 | }, 6 | "functions": { 7 | "predeploy": [ 8 | "yarn --cwd \"$RESOURCE_DIR\" lint", 9 | "yarn --cwd \"$RESOURCE_DIR\" test", 10 | "yarn --cwd \"$RESOURCE_DIR\" build" 11 | ] 12 | }, 13 | "hosting": { 14 | "public": "public", 15 | "ignore": [ 16 | "firebase.json", 17 | "**/.*", 18 | "**/node_modules/**" 19 | ] 20 | }, 21 | "storage": { 22 | "rules": "storage.rules" 23 | }, 24 | "emulators": { 25 | "functions": { 26 | "port": 5001 27 | }, 28 | "firestore": { 29 | "port": 8080 30 | }, 31 | "hosting": { 32 | "port": 5002 33 | }, 34 | "pubsub": { 35 | "port": 8085 36 | }, 37 | "ui": { 38 | "enabled": true 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /firestore.indexes.json: -------------------------------------------------------------------------------- 1 | { 2 | "indexes": [ 3 | { 4 | "collectionGroup": "ciBuilds", 5 | "queryScope": "COLLECTION", 6 | "fields": [ 7 | { 8 | "fieldPath": "relatedJobId", 9 | "order": "ASCENDING" 10 | }, 11 | { 12 | "fieldPath": "status", 13 | "order": "ASCENDING" 14 | } 15 | ] 16 | }, 17 | { 18 | "collectionGroup": "ciJobs", 19 | "queryScope": "COLLECTION", 20 | "fields": [ 21 | { 22 | "fieldPath": "repoVersionInfo.version", 23 | "order": "ASCENDING" 24 | }, 25 | { 26 | "fieldPath": "editorVersionInfo.major", 27 | "order": "DESCENDING" 28 | }, 29 | { 30 | "fieldPath": "editorVersionInfo.minor", 31 | "order": "DESCENDING" 32 | }, 33 | { 34 | "fieldPath": "editorVersionInfo.patch", 35 | "order": "DESCENDING" 36 | } 37 | ] 38 | }, 39 | { 40 | "collectionGroup": "ciJobs", 41 | "queryScope": "COLLECTION", 42 | "fields": [ 43 | { 44 | "fieldPath": "status", 45 | "order": "ASCENDING" 46 | }, 47 | { 48 | "fieldPath": "editorVersionInfo.major", 49 | "order": "DESCENDING" 50 | }, 51 | { 52 | "fieldPath": "editorVersionInfo.minor", 53 | "order": "DESCENDING" 54 | }, 55 | { 56 | "fieldPath": "editorVersionInfo.patch", 57 | "order": "DESCENDING" 58 | } 59 | ] 60 | }, 61 | { 62 | "collectionGroup": "ciJobs", 63 | "queryScope": "COLLECTION", 64 | "fields": [ 65 | { 66 | "fieldPath": "status", 67 | "order": "ASCENDING" 68 | }, 69 | { 70 | "fieldPath": "repoVersionInfo.version", 71 | "order": "ASCENDING" 72 | } 73 | ] 74 | }, 75 | { 76 | "collectionGroup": "editorVersions", 77 | "queryScope": "COLLECTION", 78 | "fields": [ 79 | { 80 | "fieldPath": "major", 81 | "order": "DESCENDING" 82 | }, 83 | { 84 | "fieldPath": "minor", 85 | "order": "DESCENDING" 86 | }, 87 | { 88 | "fieldPath": "patch", 89 | "order": "DESCENDING" 90 | } 91 | ] 92 | }, 93 | { 94 | "collectionGroup": "repoVersions", 95 | "queryScope": "COLLECTION", 96 | "fields": [ 97 | { 98 | "fieldPath": "major", 99 | "order": "DESCENDING" 100 | }, 101 | { 102 | "fieldPath": "minor", 103 | "order": "DESCENDING" 104 | }, 105 | { 106 | "fieldPath": "patch", 107 | "order": "DESCENDING" 108 | } 109 | ] 110 | } 111 | ], 112 | "fieldOverrides": [] 113 | } 114 | -------------------------------------------------------------------------------- /firestore.rules: -------------------------------------------------------------------------------- 1 | rules_version = '2'; 2 | service cloud.firestore { 3 | match /databases/{database}/documents { 4 | 5 | // 6 | // Global settings 7 | // 8 | 9 | match /{document=**} { 10 | allow read: if false; 11 | allow write: if false; 12 | } 13 | 14 | // 15 | // Per collection rules 16 | // 17 | 18 | match /ciBuilds/{ciBuild} { 19 | allow read: if true; 20 | } 21 | 22 | match /ciJobs/{ciJob} { 23 | allow read: if true; 24 | } 25 | 26 | match /editorVersions/{editorVersion} { 27 | allow read: if false; 28 | } 29 | 30 | match /repoVersions/{repoVersion} { 31 | allow read: if true; 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /functions/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | browser: true, 4 | es6: true, 5 | node: true, 6 | }, 7 | extends: ['plugin:import/errors', 'plugin:import/warnings'], 8 | parser: '@typescript-eslint/parser', 9 | parserOptions: { 10 | project: 'tsconfig.json', 11 | sourceType: 'module', 12 | }, 13 | plugins: ['@typescript-eslint', 'import'], 14 | rules: { 15 | '@typescript-eslint/adjacent-overload-signatures': 'error', 16 | '@typescript-eslint/no-empty-function': 'error', 17 | '@typescript-eslint/no-empty-interface': 'warn', 18 | '@typescript-eslint/no-floating-promises': 'error', 19 | '@typescript-eslint/no-namespace': 'error', 20 | '@typescript-eslint/no-unnecessary-type-assertion': 'error', 21 | '@typescript-eslint/prefer-for-of': 'warn', 22 | '@typescript-eslint/triple-slash-reference': 'error', 23 | '@typescript-eslint/unified-signatures': 'warn', 24 | "@typescript-eslint/no-redeclare": [ 25 | "error", 26 | { 27 | "ignoreDeclarationMerge": true, 28 | } 29 | ], 30 | 'comma-dangle': ['warn', 'always-multiline'], 31 | 'constructor-super': 'error', 32 | eqeqeq: ['warn', 'always'], 33 | 'import/no-unresolved': 0, 34 | 'import/no-deprecated': 'warn', 35 | 'import/no-extraneous-dependencies': 'error', 36 | 'import/no-unassigned-import': 'warn', 37 | 'no-cond-assign': 'error', 38 | 'no-duplicate-case': 'error', 39 | 'no-duplicate-imports': 'error', 40 | 'no-empty': [ 41 | 'warn', 42 | { 43 | allowEmptyCatch: true, 44 | }, 45 | ], 46 | 'no-invalid-this': 'error', 47 | 'no-new-wrappers': 'error', 48 | 'no-param-reassign': 'error', 49 | 'no-sequences': 'error', 50 | 'no-shadow': [ 51 | 'error', 52 | { 53 | hoist: 'all', 54 | }, 55 | ], 56 | 'no-throw-literal': 'error', 57 | 'no-unsafe-finally': 'error', 58 | 'no-unused-labels': 'error', 59 | 'no-var': 'warn', 60 | 'no-void': 'error', 61 | 'prefer-const': 'warn', 62 | }, 63 | settings: { 64 | jsdoc: { 65 | tagNamePreference: { 66 | returns: 'return', 67 | }, 68 | }, 69 | }, 70 | }; 71 | -------------------------------------------------------------------------------- /functions/.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled JavaScript files 2 | lib/**/*.js 3 | lib/**/*.js.map 4 | 5 | # TypeScript v1 declaration files 6 | typings/ 7 | 8 | # Node.js dependency directory 9 | node_modules/ 10 | 11 | # Service account 12 | service-account.json 13 | 14 | # Private key 15 | *.pem 16 | -------------------------------------------------------------------------------- /functions/.yarnrc.yml: -------------------------------------------------------------------------------- 1 | nodeLinker: node-modules 2 | -------------------------------------------------------------------------------- /functions/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "functions", 3 | "scripts": { 4 | "lint": "eslint \"src/**/*\"", 5 | "build": "tsc", 6 | "watch": "tsc --watch", 7 | "serve": "npm run build && firebase emulators:start --only functions", 8 | "shell": "npm run build && firebase functions:shell", 9 | "start": "npm run shell", 10 | "deploy": "firebase deploy --only functions", 11 | "logs": "firebase functions:log", 12 | "test": "echo \"No tests yet, feel free to add\" && exit 0" 13 | }, 14 | "engines": { 15 | "node": "20" 16 | }, 17 | "main": "lib/index.js", 18 | "dependencies": { 19 | "@octokit/auth-app": "^6.1.3", 20 | "@octokit/rest": "^20.1.1", 21 | "eris": "^0.17.2", 22 | "firebase-admin": "^12.7.0", 23 | "firebase-functions": "^6.1.0", 24 | "graphql": "^16.9.0", 25 | "lodash": "^4.17.21", 26 | "node-fetch": "^2.7.0", 27 | "semver": "^7.6.3", 28 | "unity-changeset": "^2.5.0" 29 | }, 30 | "devDependencies": { 31 | "@octokit/types": "^13.5.0", 32 | "@types/lodash": "^4.17.4", 33 | "@types/node": "^22.9.0", 34 | "@types/node-fetch": "^2.6.11", 35 | "@types/semver": "^7.5.8", 36 | "@types/ws": "^8.5.13", 37 | "@typescript-eslint/eslint-plugin": "^7.12.0", 38 | "@typescript-eslint/parser": "^7.12.0", 39 | "eslint": "^8.57.0", 40 | "eslint-plugin-import": "^2.29.1", 41 | "firebase-functions-test": "^3.3.0", 42 | "jest": "^29.7.0", 43 | "typescript": "^5.6.3" 44 | }, 45 | "private": true, 46 | "volta": { 47 | "node": "20.14.0", 48 | "yarn": "1.22.22" 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /functions/src/api/index.ts: -------------------------------------------------------------------------------- 1 | export { queueStatus } from './queueStatus'; 2 | export { reportBuildFailure } from './reportBuildFailure'; 3 | export { reportNewBuild } from './reportNewBuild'; 4 | export { reportPublication } from './reportPublication'; 5 | export { unityVersions } from './unityVersions'; 6 | export { retryBuild } from './retryBuild'; 7 | export { testFunction } from './testFunction'; 8 | -------------------------------------------------------------------------------- /functions/src/api/queueStatus.ts: -------------------------------------------------------------------------------- 1 | import { onRequest, Request } from 'firebase-functions/v2/https'; 2 | import { Response } from 'express-serve-static-core'; 3 | import { CiJobs } from '../model/ciJobs'; 4 | import { CiBuilds } from '../model/ciBuilds'; 5 | 6 | export const queueStatus = onRequest(async (req: Request, res: Response) => { 7 | const jobs = await CiJobs.getAll(); 8 | const builds = await CiBuilds.getAll(); 9 | 10 | res.status(200).send({ jobs, builds }); 11 | }); 12 | -------------------------------------------------------------------------------- /functions/src/api/repoVersions.ts: -------------------------------------------------------------------------------- 1 | import { onRequest, Request } from 'firebase-functions/v2/https'; 2 | import { logger } from 'firebase-functions/v2'; 3 | import { Response } from 'express-serve-static-core'; 4 | import { RepoVersionInfo } from '../model/repoVersionInfo'; 5 | 6 | export const repoVersions = onRequest(async (request: Request, response: Response) => { 7 | try { 8 | const versions = await RepoVersionInfo.getAllIds(); 9 | 10 | response.send(versions); 11 | } catch (err) { 12 | logger.error(err); 13 | response.send('Oops.'); 14 | } 15 | }); 16 | -------------------------------------------------------------------------------- /functions/src/api/reportBuildFailure.ts: -------------------------------------------------------------------------------- 1 | import { onRequest, Request } from 'firebase-functions/v2/https'; 2 | import { logger } from 'firebase-functions/v2'; 3 | import { Response } from 'express-serve-static-core'; 4 | import { Token } from '../config/token'; 5 | import { BuildFailure, CiBuilds } from '../model/ciBuilds'; 6 | import { CiJobs } from '../model/ciJobs'; 7 | import { Discord } from '../service/discord'; 8 | import { defineSecret } from 'firebase-functions/params'; 9 | 10 | const discordToken = defineSecret('DISCORD_TOKEN'); 11 | const internalToken = defineSecret('INTERNAL_TOKEN'); 12 | 13 | export const reportBuildFailure = onRequest( 14 | { secrets: [discordToken, internalToken] }, 15 | async (req: Request, res: Response) => { 16 | await Discord.init(discordToken.value()); 17 | 18 | try { 19 | if (!Token.isValid(req.header('authorization'), internalToken.value())) { 20 | logger.warn('unauthorised request', req.headers); 21 | res.status(403).send('Unauthorized'); 22 | return; 23 | } 24 | 25 | const { body } = req; 26 | logger.debug('Build failure report incoming.', body); 27 | 28 | const { jobId, buildId, reason } = body; 29 | const failure: BuildFailure = { reason }; 30 | 31 | await CiJobs.markFailureForJob(jobId); 32 | await CiBuilds.markBuildAsFailed(buildId, failure); 33 | 34 | logger.info('Build failure reported.', body); 35 | res.status(200).send('OK'); 36 | } catch (err: any) { 37 | const message = ` 38 | Something went wrong while reporting a build failure 39 | ${err.message} 40 | `; 41 | logger.error(message, err); 42 | 43 | await Discord.sendAlert(message); 44 | 45 | if (req.body?.jobId?.toString().startsWith('dryRun')) { 46 | await CiBuilds.removeDryRunBuild(req.body.buildId); 47 | await CiJobs.removeDryRunJob(req.body.jobId); 48 | } 49 | 50 | res.status(500).send('Something went wrong'); 51 | } 52 | 53 | Discord.disconnect(); 54 | }, 55 | ); 56 | -------------------------------------------------------------------------------- /functions/src/api/reportNewBuild.ts: -------------------------------------------------------------------------------- 1 | import { onRequest, Request } from 'firebase-functions/v2/https'; 2 | import { logger } from 'firebase-functions/v2'; 3 | import { Response } from 'express-serve-static-core'; 4 | import { Token } from '../config/token'; 5 | import { BuildInfo, CiBuilds } from '../model/ciBuilds'; 6 | import { CiJobs } from '../model/ciJobs'; 7 | import { Discord } from '../service/discord'; 8 | import { EditorVersionInfo } from '../model/editorVersionInfo'; 9 | import { RepoVersionInfo } from '../model/repoVersionInfo'; 10 | import { Image, ImageType } from '../model/image'; 11 | import { defineSecret } from 'firebase-functions/params'; 12 | 13 | const discordToken = defineSecret('DISCORD_TOKEN'); 14 | const internalToken = defineSecret('INTERNAL_TOKEN'); 15 | 16 | export const reportNewBuild = onRequest( 17 | { secrets: [discordToken, internalToken] }, 18 | async (req: Request, res: Response) => { 19 | await Discord.init(discordToken.value()); 20 | 21 | try { 22 | if (!Token.isValid(req.header('authorization'), internalToken.value())) { 23 | logger.warn('unauthorised request', req.headers); 24 | res.status(403).send('Unauthorized'); 25 | return; 26 | } 27 | 28 | const { body } = req; 29 | logger.debug('new incoming build report', body); 30 | 31 | const { buildId, jobId, imageType, baseOs, repoVersion, editorVersion, targetPlatform } = 32 | body; 33 | const buildInfo: BuildInfo = { 34 | baseOs, 35 | repoVersion, 36 | editorVersion, 37 | targetPlatform, 38 | }; 39 | 40 | if (jobId.toString().startsWith('dryRun')) { 41 | await createDryRunJob(jobId, imageType, editorVersion); 42 | } 43 | 44 | await CiJobs.markJobAsInProgress(jobId); 45 | await CiBuilds.registerNewBuild(buildId, jobId, imageType, buildInfo); 46 | 47 | logger.info('new build reported', body); 48 | res.status(200).send('OK'); 49 | } catch (err: any) { 50 | const message = ` 51 | Something went wrong while wrong while reporting a new build. 52 | ${err.message} 53 | `; 54 | logger.error(message, err); 55 | await Discord.sendAlert(message); 56 | 57 | if (req.body?.jobId?.toString().startsWith('dryRun')) { 58 | await CiBuilds.removeDryRunBuild(req.body.buildId); 59 | await CiJobs.removeDryRunJob(req.body.jobId); 60 | } 61 | 62 | res.status(500).send('Something went wrong'); 63 | 64 | Discord.disconnect(); 65 | } 66 | }, 67 | ); 68 | 69 | const createDryRunJob = async (jobId: string, imageType: ImageType, editorVersion: string) => { 70 | logger.debug('running dryrun for image', imageType, editorVersion); 71 | const repoVersionInfo = await RepoVersionInfo.getLatest(); 72 | 73 | if (imageType === Image.types.editor) { 74 | const editorVersionInfo = await EditorVersionInfo.get(editorVersion); 75 | await CiJobs.create(jobId, imageType, repoVersionInfo, editorVersionInfo); 76 | } else { 77 | await CiJobs.create(jobId, imageType, repoVersionInfo); 78 | } 79 | }; 80 | -------------------------------------------------------------------------------- /functions/src/api/reportPublication.ts: -------------------------------------------------------------------------------- 1 | import { onRequest, Request } from 'firebase-functions/v2/https'; 2 | import { logger } from 'firebase-functions/v2'; 3 | import { Response } from 'express-serve-static-core'; 4 | import { Token } from '../config/token'; 5 | import { CiBuilds } from '../model/ciBuilds'; 6 | import { CiJobs } from '../model/ciJobs'; 7 | import { Discord } from '../service/discord'; 8 | import { Image } from '../model/image'; 9 | import { defineSecret } from 'firebase-functions/params'; 10 | 11 | const discordToken = defineSecret('DISCORD_TOKEN'); 12 | const internalToken = defineSecret('INTERNAL_TOKEN'); 13 | 14 | export const reportPublication = onRequest( 15 | { secrets: [discordToken, internalToken] }, 16 | async (req: Request, res: Response) => { 17 | await Discord.init(discordToken.value()); 18 | 19 | try { 20 | if (!Token.isValid(req.header('authorization'), internalToken.value())) { 21 | logger.warn('unauthorised request', req.headers); 22 | res.status(403).send('Unauthorized'); 23 | return; 24 | } 25 | 26 | const { body } = req; 27 | logger.debug('Publication report incoming.', body); 28 | const isDryRun = req.body.jobId?.toString().startsWith('dryRun'); 29 | 30 | const { jobId, buildId, dockerInfo } = body; 31 | const parentJobIsNowCompleted = await CiBuilds.markBuildAsPublished( 32 | buildId, 33 | jobId, 34 | dockerInfo, 35 | ); 36 | if (parentJobIsNowCompleted) { 37 | // Report new publications as news 38 | let message; 39 | if (dockerInfo.imageName === Image.types.editor) { 40 | // i.e. [editor-2017.1.0f3-0.5.0] 41 | const [imageType, publicationName, version] = jobId.split('-'); 42 | // i.e. [v0.5.0] images for [editor] [2020.2.22f2] 43 | message = `Published v${version} images for ${imageType} ${publicationName}.`; 44 | } else { 45 | // i.e. [hub-0.5.0] 46 | const [publicationName, version] = jobId.split('-'); 47 | // i.e. [hub] or [base] for v0.5.0 48 | message = `New ${publicationName}-image published for v${version}.`; 49 | } 50 | logger.info(message); 51 | if (!isDryRun) { 52 | await Discord.sendNews(message); 53 | } 54 | } 55 | 56 | logger.info('Publication reported.', body); 57 | if (isDryRun) { 58 | await CiBuilds.removeDryRunBuild(req.body.buildId); 59 | await CiJobs.removeDryRunJob(req.body.jobId); 60 | } 61 | 62 | res.status(200).send('OK'); 63 | } catch (err: any) { 64 | const message = ` 65 | Something went wrong while wrong while reporting a new publication 66 | ${err.message} 67 | `; 68 | logger.error(message, err); 69 | await Discord.sendAlert(message); 70 | 71 | if (req.body?.jobId?.toString().startsWith('dryRun')) { 72 | await CiBuilds.removeDryRunBuild(req.body.buildId); 73 | await CiJobs.removeDryRunJob(req.body.jobId); 74 | } 75 | 76 | res.status(500).send('Something went wrong'); 77 | } 78 | 79 | await Discord.disconnect(); 80 | }, 81 | ); 82 | -------------------------------------------------------------------------------- /functions/src/api/retryBuild.ts: -------------------------------------------------------------------------------- 1 | import { admin } from '../service/firebase'; 2 | import { onRequest, Request } from 'firebase-functions/v2/https'; 3 | import { Response } from 'express-serve-static-core'; 4 | import { CiBuilds } from '../model/ciBuilds'; 5 | import { CiJobs } from '../model/ciJobs'; 6 | import { Ingeminator } from '../logic/buildQueue/ingeminator'; 7 | import { GitHub } from '../service/github'; 8 | import { RepoVersionInfo } from '../model/repoVersionInfo'; 9 | import { Discord } from '../service/discord'; 10 | import { defineSecret } from 'firebase-functions/params'; 11 | 12 | const discordToken = defineSecret('DISCORD_TOKEN'); 13 | const githubPrivateKey = defineSecret('GITHUB_PRIVATE_KEY'); 14 | const githubClientSecret = defineSecret('GITHUB_CLIENT_SECRET'); 15 | 16 | export const retryBuild = onRequest( 17 | { secrets: [discordToken, githubClientSecret, githubPrivateKey] }, 18 | async (request: Request, response: Response) => { 19 | await Discord.init(discordToken.value()); 20 | 21 | try { 22 | response.set('Content-Type', 'application/json'); 23 | 24 | // Allow pre-flight from cross origin 25 | response.set('Access-Control-Allow-Origin', '*'); 26 | response.set('Access-Control-Allow-Methods', ['POST']); 27 | response.set('Access-Control-Allow-Headers', ['Content-Type', 'Authorization']); 28 | if (request.method === 'OPTIONS') { 29 | response.status(204).send({ message: 'OK' }); 30 | return; 31 | } 32 | 33 | // User must be authenticated 34 | const token = request.header('Authorization')?.replace(/^Bearer\s/, ''); 35 | console.log('token:', token); 36 | if (!token) { 37 | response.status(401).send({ message: 'Unauthorized' }); 38 | return; 39 | } 40 | 41 | // User must be an admin 42 | const user = await admin.auth().verifyIdToken(token); 43 | if (!user || !user.email_verified || !user.admin) { 44 | response.status(401).send({ message: 'Unauthorized' }); 45 | return; 46 | } 47 | 48 | // Validate arguments 49 | const { buildId, relatedJobId: jobId } = request.body; 50 | if (!buildId || !jobId) { 51 | response.status(400); 52 | response.send({ 53 | message: 'Bad request', 54 | description: 'Expected buildId and relatedJobId.', 55 | }); 56 | return; 57 | } 58 | 59 | // Only retry existing builds 60 | const [job, build] = await Promise.all([CiJobs.get(jobId), CiBuilds.get(buildId)]); 61 | if (!job || !build) { 62 | response.status(400); 63 | response.send({ 64 | message: 'Bad request', 65 | description: 'Expected valid buildId and relatedJobId.', 66 | }); 67 | return; 68 | } 69 | 70 | // Check if build is not already running 71 | if (build.status === 'started') { 72 | response.status(409); 73 | response.send({ 74 | message: 'Build was already re-scheduled', 75 | description: request.body.buildId, 76 | }); 77 | return; 78 | } 79 | 80 | // Schedule new build 81 | const gitHubClient = await GitHub.init(githubPrivateKey.value(), githubClientSecret.value()); 82 | const repoVersionInfo = await RepoVersionInfo.getLatest(); 83 | const scheduler = new Ingeminator(1, gitHubClient, repoVersionInfo); 84 | const scheduledSuccessfully = await scheduler.rescheduleBuild(jobId, job, buildId, build); 85 | 86 | // Report result 87 | if (scheduledSuccessfully) { 88 | response.status(200); 89 | response.send({ 90 | message: 'Build has been rescheduled', 91 | description: request.body.buildId, 92 | }); 93 | } else { 94 | response.status(408); 95 | response.send({ 96 | message: 'Request to Dockerhub timed out', 97 | description: request.body.buildId, 98 | }); 99 | } 100 | } catch (error) { 101 | console.log('error', JSON.stringify(error, Object.getOwnPropertyNames(error))); 102 | response.status(401).send({ message: 'Unauthorized' }); 103 | } 104 | 105 | Discord.disconnect(); 106 | }, 107 | ); 108 | -------------------------------------------------------------------------------- /functions/src/api/testFunction.ts: -------------------------------------------------------------------------------- 1 | import { onRequest, Request } from 'firebase-functions/v2/https'; 2 | import { Response } from 'express-serve-static-core'; 3 | import { defineSecret } from 'firebase-functions/params'; 4 | import { scrapeVersions } from '../logic/ingestRepoVersions/scrapeVersions'; 5 | import { scrapeVersions as scrapeUnityVersions } from '../logic/ingestUnityVersions/scrapeVersions'; 6 | 7 | import { Discord } from '../service/discord'; 8 | 9 | const discordToken = defineSecret('DISCORD_TOKEN'); 10 | const githubPrivateKeyConfigSecret = defineSecret('GITHUB_PRIVATE_KEY'); 11 | const githubClientSecretConfigSecret = defineSecret('GITHUB_CLIENT_SECRET'); 12 | const internalToken = defineSecret('INTERNAL_TOKEN'); 13 | 14 | export const testFunction = onRequest( 15 | { 16 | // Passing all secrets so that test deployments verify that the secrets are correctly set. 17 | secrets: [ 18 | discordToken, 19 | githubPrivateKeyConfigSecret, 20 | githubClientSecretConfigSecret, 21 | internalToken, 22 | ], 23 | }, 24 | async (request: Request, response: Response) => { 25 | // Run all non-sensitive functions to verify that the deployment is working. 26 | let info = 'Ok'; 27 | let code = 200; 28 | 29 | try { 30 | await Discord.init(discordToken.value()); 31 | 32 | const versions = await scrapeVersions( 33 | githubPrivateKeyConfigSecret.value(), 34 | githubClientSecretConfigSecret.value(), 35 | ); 36 | 37 | if (versions.length === 0) { 38 | throw new Error('No versions were found.'); 39 | } 40 | 41 | const unityVersions = await scrapeUnityVersions(); 42 | if (unityVersions.length === 0) { 43 | throw new Error('No Unity versions were found.'); 44 | } 45 | 46 | info = `Found ${versions.length} repo versions and ${ 47 | unityVersions.length 48 | } Unity versions. Unity Versions: \n${unityVersions 49 | .map((unity) => `${unity.version}:${unity.changeSet}`) 50 | .join('\n')}`; 51 | } catch (error: any) { 52 | info = error.message; 53 | code = 500; 54 | } finally { 55 | await Discord.disconnect(); 56 | } 57 | 58 | response.status(code).send(info); 59 | }, 60 | ); 61 | -------------------------------------------------------------------------------- /functions/src/api/unityVersions.ts: -------------------------------------------------------------------------------- 1 | import { onRequest, Request } from 'firebase-functions/v2/https'; 2 | import { Response } from 'express-serve-static-core'; 3 | import { EditorVersionInfo } from '../model/editorVersionInfo'; 4 | import { logger } from 'firebase-functions/v2'; 5 | 6 | export const unityVersions = onRequest(async (request: Request, response: Response) => { 7 | try { 8 | const versions = await EditorVersionInfo.getAllIds(); 9 | 10 | response.send(versions); 11 | } catch (err) { 12 | logger.error(err); 13 | response.send('Oops.'); 14 | } 15 | }); 16 | -------------------------------------------------------------------------------- /functions/src/config/credential.ts: -------------------------------------------------------------------------------- 1 | import { credential } from 'firebase-admin'; 2 | 3 | export const getCredential = (): credential.Credential => { 4 | try { 5 | const serviceAccount = require(process.env.GOOGLE_APPLICATION_CREDENTIALS ?? 6 | '../../service-account.json'); 7 | return credential.cert(serviceAccount); 8 | } catch (e) { 9 | return credential.applicationDefault(); 10 | } 11 | }; 12 | -------------------------------------------------------------------------------- /functions/src/config/settings.ts: -------------------------------------------------------------------------------- 1 | export const settings = { 2 | defaultAdmins: [ 3 | 'webber.nl@gmail.com', 4 | 'lebreton.gabriel@gmail.com', 5 | 'davidmfinol@gmail.com', 6 | 'andrewk010110@gmail.com', 7 | ], 8 | minutesBetweenScans: 15, 9 | maxConcurrentJobs: 9, 10 | maxExtraJobsForRescheduling: 1, 11 | maxToleratedFailures: 2, 12 | maxFailuresPerBuild: 15, 13 | discord: { 14 | channels: { 15 | news: '731947345478156391', // #news (public) 16 | maintainers: '764289922663841792', // # build-notifications (internal) 17 | alerts: '763544776649605151', // #alerts (internal) 18 | debug: '815382556143910953', // #backend (internal) 19 | }, 20 | }, 21 | github: { 22 | auth: { 23 | appId: 84327, 24 | installationId: 12321333, 25 | clientId: 'Iv1.fa93dce6a47c9357', 26 | }, 27 | }, 28 | dockerhub: { 29 | host: 'https://index.docker.io/v1', 30 | repositoryBaseName: 'unityci', 31 | baseImageName: 'base', 32 | hubImageName: 'hub', 33 | editorImageName: 'editor', 34 | }, 35 | }; 36 | -------------------------------------------------------------------------------- /functions/src/config/token.ts: -------------------------------------------------------------------------------- 1 | export class Token { 2 | static isValid(providedToken: string | null | undefined, internalToken: string) { 3 | if (!providedToken) return false; 4 | 5 | return providedToken === internalToken || providedToken === `Bearer ${internalToken}`; 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /functions/src/cron/index.ts: -------------------------------------------------------------------------------- 1 | import { Discord } from '../service/discord'; 2 | import { ingestUnityVersions } from '../logic/ingestUnityVersions'; 3 | import { ingestRepoVersions } from '../logic/ingestRepoVersions'; 4 | import { cleanUpBuilds, scheduleBuildsFromTheQueue } from '../logic/buildQueue'; 5 | import { settings } from '../config/settings'; 6 | import { dataMigrations } from '../logic/dataTransformation'; 7 | import { onSchedule } from 'firebase-functions/v2/scheduler'; 8 | import { logger } from 'firebase-functions/v2'; 9 | import { defineSecret } from 'firebase-functions/params'; 10 | 11 | const discordToken = defineSecret('DISCORD_TOKEN'); 12 | const githubPrivateKeyConfigSecret = defineSecret('GITHUB_PRIVATE_KEY'); 13 | const githubClientSecretConfigSecret = defineSecret('GITHUB_CLIENT_SECRET'); 14 | 15 | const MINUTES: number = settings.minutesBetweenScans; 16 | if (MINUTES < 10) { 17 | throw new Error('Is the result really worth the machine time? Remove me.'); 18 | } 19 | 20 | // Timeout of 60 seconds will keep our routine process tight. 21 | export const trigger = onSchedule( 22 | { 23 | schedule: `every ${MINUTES} minutes`, 24 | memory: '512MiB', 25 | timeoutSeconds: 60, 26 | secrets: [discordToken, githubPrivateKeyConfigSecret, githubClientSecretConfigSecret], 27 | }, 28 | async () => { 29 | await Discord.init(discordToken.value()); 30 | 31 | try { 32 | await routineTasks( 33 | githubPrivateKeyConfigSecret.value(), 34 | githubClientSecretConfigSecret.value(), 35 | ); 36 | } catch (error: any) { 37 | const errorStatus = error.status ? ` (${error.status})` : ''; 38 | const errorStack = error.stackTrace ? `\n${error.stackTrace}` : ''; 39 | const fullError = `${error.message}${errorStatus}${errorStack}`; 40 | 41 | const routineTasksFailedMessage = `Something went wrong while running routine tasks.\n${fullError}`; 42 | 43 | logger.error(routineTasksFailedMessage); 44 | await Discord.sendAlert(routineTasksFailedMessage); 45 | } 46 | 47 | Discord.disconnect(); 48 | }, 49 | ); 50 | 51 | const routineTasks = async (githubPrivateKey: string, githubClientSecret: string) => { 52 | try { 53 | await Discord.sendDebugLine('begin'); 54 | await dataMigrations(); 55 | await ingestRepoVersions(githubPrivateKey, githubClientSecret); 56 | await ingestUnityVersions(); 57 | await cleanUpBuilds(); 58 | await scheduleBuildsFromTheQueue(githubPrivateKey, githubClientSecret); 59 | } catch (error: any) { 60 | logger.error(error); 61 | await Discord.sendAlert(error); 62 | } finally { 63 | await Discord.sendDebugLine('end'); 64 | } 65 | }; 66 | -------------------------------------------------------------------------------- /functions/src/index.ts: -------------------------------------------------------------------------------- 1 | import { settings } from './config/settings'; 2 | import { assignDefaultAdmins } from './logic/init/assignDefaultAdmins'; 3 | 4 | const init = async () => { 5 | await assignDefaultAdmins(settings.defaultAdmins); 6 | }; 7 | 8 | init().catch((error) => { 9 | console.error(error); 10 | throw new Error('Deployment failed due to errors.'); 11 | }); 12 | 13 | export * as model from './model-triggers'; 14 | export * as cron from './cron'; 15 | export * from './api'; 16 | -------------------------------------------------------------------------------- /functions/src/logic/buildQueue/cleanUpBuilds.ts: -------------------------------------------------------------------------------- 1 | import { Cleaner } from './cleaner'; 2 | 3 | export const cleanUpBuilds = async () => { 4 | await Cleaner.cleanUp(); 5 | }; 6 | -------------------------------------------------------------------------------- /functions/src/logic/buildQueue/cleaner.ts: -------------------------------------------------------------------------------- 1 | import { CiBuilds } from '../../model/ciBuilds'; 2 | import { Discord } from '../../service/discord'; 3 | import { Dockerhub } from '../../service/dockerhub'; 4 | 5 | export class Cleaner { 6 | // Cronjob intentionally has a limited runtime 7 | static readonly maxBuildsProcessedPerRun: number = 5; 8 | 9 | static buildsProcessed: number; 10 | 11 | public static async cleanUp() { 12 | this.buildsProcessed = 0; 13 | await this.cleanUpBuildsThatDidntReportBack(); 14 | } 15 | 16 | private static async cleanUpBuildsThatDidntReportBack() { 17 | const startedBuilds = await CiBuilds.getStartedBuilds(); 18 | 19 | for (const startedBuild of startedBuilds) { 20 | if (this.buildsProcessed >= this.maxBuildsProcessedPerRun) return; 21 | 22 | const { buildId, meta, relatedJobId: jobId, imageType, buildInfo } = startedBuild; 23 | const { publishedDate, lastBuildStart } = meta; 24 | const { baseOs, repoVersion } = buildInfo; 25 | 26 | const tag = buildId.replace(new RegExp(`^${imageType}-`), ''); 27 | 28 | if (publishedDate) { 29 | const buildWasPublishedAlreadyMessage = `[Cleaner] Build "${tag}" has a publication date, but it's status is "started". Was a rebuild requested for this build?`; 30 | await Discord.sendDebug(buildWasPublishedAlreadyMessage); 31 | 32 | // Maybe set status to published. However, that will increase complexity. 33 | // Deleting a tag from dockerhub and rebuilding will yield this error currently. 34 | // If we set it to published, we also need to look up the build digest from dockerhub. 35 | 36 | continue; 37 | } 38 | 39 | if (!lastBuildStart) { 40 | // In theory this should never happen. 41 | await Discord.sendAlert( 42 | `[Cleaner] Build "${tag}" with status "started" does not have a "lastBuildStart" date.`, 43 | ); 44 | continue; 45 | } 46 | 47 | // Job execution time - Each job in a workflow can run for up to 6 hours of execution time. 48 | // If a job reaches this limit, the job is terminated and fails to complete. 49 | // @see https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration 50 | const ONE_HOUR = 1000 * 60 * 60; 51 | const sixHoursAgo = new Date().getTime() - 6 * ONE_HOUR; 52 | const buildStart = new Date(lastBuildStart.seconds * 1000).getTime(); 53 | 54 | if (buildStart < sixHoursAgo) { 55 | this.buildsProcessed += 1; 56 | 57 | const response = await Dockerhub.fetchImageData(imageType, tag); 58 | 59 | // Image does not exist 60 | if (!response) { 61 | const markAsFailedMessage = `[Cleaner] Build for "${tag}" with status "started" never reported back in. Marking it as failed. It will retry automatically.`; 62 | await Discord.sendAlert(markAsFailedMessage); 63 | await CiBuilds.markBuildAsFailed(buildId, { 64 | reason: markAsFailedMessage, 65 | }); 66 | 67 | continue; 68 | } 69 | 70 | // Image exists 71 | const markAsSuccessfulMessage = `[Cleaner] Build for "${tag}" got stuck. But the image was successfully uploaded. Marking it as published.`; 72 | await Discord.sendDebug(markAsSuccessfulMessage); 73 | await CiBuilds.markBuildAsPublished(buildId, jobId, { 74 | digest: '', // missing from dockerhub v1 api payload 75 | specificTag: `${baseOs}-${repoVersion}`, 76 | friendlyTag: repoVersion.replace(/\.\d+$/, ''), 77 | imageName: Dockerhub.getImageName(imageType), 78 | imageRepo: Dockerhub.getRepositoryBaseName(), 79 | }); 80 | 81 | continue; 82 | } 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /functions/src/logic/buildQueue/index.ts: -------------------------------------------------------------------------------- 1 | export { cleanUpBuilds } from './cleanUpBuilds'; 2 | export { scheduleBuildsFromTheQueue } from './scheduleBuildsFromTheQueue'; 3 | -------------------------------------------------------------------------------- /functions/src/logic/buildQueue/ingeminator.ts: -------------------------------------------------------------------------------- 1 | import { CiJob, CiJobQueue, CiJobQueueItem, CiJobs } from '../../model/ciJobs'; 2 | import { CiBuild, CiBuilds } from '../../model/ciBuilds'; 3 | import { EditorVersionInfo } from '../../model/editorVersionInfo'; 4 | import { Discord } from '../../service/discord'; 5 | import { Octokit } from '@octokit/rest'; 6 | import { RepoVersionInfo } from '../../model/repoVersionInfo'; 7 | import { Scheduler } from './scheduler'; 8 | import admin from 'firebase-admin'; 9 | import Timestamp = admin.firestore.Timestamp; 10 | import { settings } from '../../config/settings'; 11 | import { GitHubWorkflow } from '../../model/gitHubWorkflow'; 12 | import { Image } from '../../model/image'; 13 | import { logger } from 'firebase-functions/v2'; 14 | 15 | export class Ingeminator { 16 | numberToSchedule: number; 17 | gitHubClient: Octokit; 18 | repoVersionInfo: RepoVersionInfo; 19 | 20 | constructor(numberToSchedule: number, gitHubClient: Octokit, repoVersionInfo: RepoVersionInfo) { 21 | this.numberToSchedule = numberToSchedule; 22 | this.gitHubClient = gitHubClient; 23 | this.repoVersionInfo = repoVersionInfo; 24 | } 25 | 26 | async rescheduleFailedJobs(jobs: CiJobQueue) { 27 | if (jobs.length <= 0) { 28 | throw new Error( 29 | '[Ingeminator] Expected ingeminator to be called with jobs to retry, none were given.', 30 | ); 31 | } 32 | 33 | for (const job of jobs) { 34 | if (job.data.imageType !== Image.types.editor) { 35 | throw new Error( 36 | '[Ingeminator] Did not expect to be handling non-editor image type rescheduling.', 37 | ); 38 | } 39 | 40 | await this.rescheduleFailedBuildsForJob(job); 41 | } 42 | } 43 | 44 | private async rescheduleFailedBuildsForJob(job: CiJobQueueItem) { 45 | const { id: jobId, data: jobData } = job; 46 | const builds = await CiBuilds.getFailedBuildsQueue(jobId); 47 | if (builds.length <= 0) { 48 | await Discord.sendDebug( 49 | `[Ingeminator] Looks like all failed builds for job \`${jobId}\` are already scheduled.`, 50 | ); 51 | return; 52 | } 53 | 54 | for (const build of builds) { 55 | const { id: buildId, data: BuildData } = build; 56 | 57 | // Space for more? 58 | if (this.numberToSchedule <= 0) { 59 | await Discord.sendDebug( 60 | `[Ingeminator] waiting for more spots to become available for builds of ${jobId}.`, 61 | ); 62 | return; 63 | } 64 | 65 | // Max retries check 66 | const { maxFailuresPerBuild } = settings; 67 | const { lastBuildFailure, failureCount } = build.data.meta; 68 | const lastFailure = lastBuildFailure as Timestamp; 69 | if (failureCount >= maxFailuresPerBuild) { 70 | // Log warning 71 | const retries: number = maxFailuresPerBuild - 1; 72 | const maxRetriesReachedMessage = 73 | `[Ingeminator] Reached the maximum amount of retries (${retries}) for ${buildId}.` + 74 | `Manual action is now required. Visit https://console.firebase.google.com/u/0/project/unity-ci-versions/firestore/data/~2FciBuilds~2F${buildId}`; 75 | 76 | // Only send alert to discord once 77 | const alertingPeriodMinutes = settings.minutesBetweenScans; 78 | const alertingPeriodMilliseconds = alertingPeriodMinutes * 60 * 1000; 79 | if (lastFailure.toMillis() + alertingPeriodMilliseconds >= Timestamp.now().toMillis()) { 80 | logger.error(maxRetriesReachedMessage); 81 | await Discord.sendAlert(maxRetriesReachedMessage); 82 | } else { 83 | await Discord.sendDebug(maxRetriesReachedMessage); 84 | } 85 | 86 | return; 87 | } 88 | 89 | // Incremental backoff 90 | const backoffMinutes = failureCount * 15; 91 | const backoffMilliseconds = backoffMinutes * 60 * 1000; 92 | if (lastFailure.toMillis() + backoffMilliseconds >= Timestamp.now().toMillis()) { 93 | await Discord.sendDebug( 94 | `[Ingeminator] Backoff period of ${backoffMinutes} minutes has not expired for ${buildId}.`, 95 | ); 96 | continue; 97 | } 98 | 99 | // Schedule a build 100 | this.numberToSchedule -= 1; 101 | if (!(await this.rescheduleBuild(jobId, jobData, buildId, BuildData))) { 102 | return; 103 | } 104 | } 105 | 106 | await CiJobs.markJobAsScheduled(jobId); 107 | await Discord.sendDebug(`[Ingeminator] rescheduled any failing editor images for ${jobId}.`); 108 | } 109 | 110 | public async rescheduleBuild(jobId: string, jobData: CiJob, buildId: string, buildData: CiBuild) { 111 | // Info from job 112 | const { editorVersionInfo } = jobData; 113 | const { version: editorVersion, changeSet } = editorVersionInfo as EditorVersionInfo; 114 | 115 | // Info from build 116 | const { buildInfo } = buildData; 117 | const { baseOs, targetPlatform } = buildInfo; 118 | 119 | // Info from repo 120 | const repoVersions = Scheduler.parseRepoVersions(this.repoVersionInfo); 121 | const { repoVersionFull, repoVersionMinor, repoVersionMajor } = repoVersions; 122 | 123 | // Send the retry request 124 | const response = await this.gitHubClient.repos.createDispatchEvent({ 125 | owner: 'unity-ci', 126 | repo: 'docker', 127 | event_type: GitHubWorkflow.getEventTypeForRetryingEditorCiBuild(baseOs), 128 | client_payload: { 129 | jobId, 130 | buildId, 131 | editorVersion, 132 | changeSet, 133 | baseOs, // specific to retry jobs 134 | targetPlatform, // specific to retry jobs 135 | repoVersionFull, 136 | repoVersionMinor, 137 | repoVersionMajor, 138 | }, 139 | }); 140 | 141 | if (response.status <= 199 || response.status >= 300) { 142 | const failureMessage = ` 143 | [Ingeminator] failed to ingeminate job ${jobId}, 144 | status: ${response.status}, response: ${response.data}.`; 145 | logger.error(failureMessage); 146 | await Discord.sendAlert(failureMessage); 147 | return false; 148 | } 149 | 150 | return true; 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /functions/src/logic/buildQueue/scheduleBuildsFromTheQueue.ts: -------------------------------------------------------------------------------- 1 | import { RepoVersionInfo } from '../../model/repoVersionInfo'; 2 | import { Scheduler } from './scheduler'; 3 | import { Discord } from '../../service/discord'; 4 | 5 | /** 6 | * When a new Unity version gets ingested: 7 | * - a CI Job for that version gets created. 8 | * 9 | * When a new repository version gets ingested 10 | * - a CI Job for a new base image gets created 11 | * - a CI Job for a new hub image gets created 12 | * - a CI Job for every Unity version gets created 13 | * - Any CI Jobs for older repository versions get status "superseded" 14 | * 15 | * This schedule is based on that knowledge and assumption 16 | */ 17 | export const scheduleBuildsFromTheQueue = async ( 18 | githubPrivateKey: string, 19 | githubClientSecret: string, 20 | ) => { 21 | const repoVersionInfo = await RepoVersionInfo.getLatest(); 22 | const scheduler = await new Scheduler(repoVersionInfo).init(githubPrivateKey, githubClientSecret); 23 | 24 | const testVersion = '0.1.0'; 25 | if (repoVersionInfo.version === testVersion) { 26 | await Discord.sendDebug('[Build queue] No longer building test versions.'); 27 | return; 28 | } 29 | 30 | if (!(await scheduler.ensureThatBaseImageHasBeenBuilt())) { 31 | await Discord.sendDebug('[Build queue] Waiting for base image to be ready.'); 32 | return; 33 | } 34 | 35 | if (!(await scheduler.ensureThatHubImageHasBeenBuilt())) { 36 | await Discord.sendDebug('[Build queue] Waiting for hub image to be ready.'); 37 | return; 38 | } 39 | 40 | if (!(await scheduler.ensureThereAreNoFailedJobs())) { 41 | await Discord.sendDebug('[Build queue] Retrying failed jobs before scheduling new jobs.'); 42 | return; 43 | } 44 | 45 | if (!(await scheduler.buildLatestEditorImages())) { 46 | await Discord.sendDebug('[Build queue] Editor images are building.'); 47 | return; 48 | } 49 | 50 | await Discord.sendDebug('[Build queue] Idle 🎈'); 51 | }; 52 | -------------------------------------------------------------------------------- /functions/src/logic/buildQueue/scheduler.ts: -------------------------------------------------------------------------------- 1 | import { RepoVersionInfo } from '../../model/repoVersionInfo'; 2 | import { GitHub } from '../../service/github'; 3 | import { Octokit } from '@octokit/rest'; 4 | import { CiJobs } from '../../model/ciJobs'; 5 | import { logger } from 'firebase-functions/v2'; 6 | import { settings } from '../../config/settings'; 7 | import { take } from 'lodash'; 8 | import { EditorVersionInfo } from '../../model/editorVersionInfo'; 9 | import { Discord } from '../../service/discord'; 10 | import { Ingeminator } from './ingeminator'; 11 | import { GitHubWorkflow } from '../../model/gitHubWorkflow'; 12 | import { Image } from '../../model/image'; 13 | 14 | export class Scheduler { 15 | private repoVersion: string; 16 | private repoVersionFull: string; 17 | private repoVersionMinor: string; 18 | private repoVersionMajor: string; 19 | private _gitHub: Octokit | undefined; 20 | private maxConcurrentJobs: number; 21 | private repoVersionInfo: RepoVersionInfo; 22 | 23 | private get gitHub(): Octokit { 24 | // @ts-ignore 25 | return this._gitHub; 26 | } 27 | 28 | constructor(repoVersionInfo: RepoVersionInfo) { 29 | this.repoVersionInfo = repoVersionInfo; 30 | this.maxConcurrentJobs = settings.maxConcurrentJobs; 31 | 32 | const { repoVersion, repoVersionFull, repoVersionMinor, repoVersionMajor } = 33 | Scheduler.parseRepoVersions(repoVersionInfo); 34 | this.repoVersion = repoVersion; 35 | this.repoVersionFull = repoVersionFull; 36 | this.repoVersionMinor = repoVersionMinor; 37 | this.repoVersionMajor = repoVersionMajor; 38 | 39 | return this; 40 | } 41 | 42 | /** 43 | * Todo - Fix regrets as needed 44 | * 45 | * Note: The names of these may be very confusing at first. 46 | * Note: This should be part of a proper model class after moving away logic from the scheduler. 47 | * Note: All refactors have to be considering the action in unityci/docker. 48 | */ 49 | static parseRepoVersions = (repoVersionInfo: RepoVersionInfo) => { 50 | const { major, minor, patch, version: repoVersion } = repoVersionInfo; 51 | // Full version tag, including patch number 52 | const repoVersionFull = `${major}.${minor}.${patch}`; 53 | // Reduced version tag, intended for allowing to pull latest minor version 54 | const repoVersionMinor = `${major}.${minor}`; 55 | // Reduced version tag, intended for people who want to have the latest version without breaking changes. 56 | const repoVersionMajor = `${major}`; 57 | 58 | if (repoVersionFull !== repoVersion) { 59 | throw new Error(` 60 | [Scheduler] Expected version information to be reliable 61 | Received ${repoVersionFull} vs ${repoVersion}`); 62 | } 63 | 64 | return { 65 | repoVersion, 66 | repoVersionFull, 67 | repoVersionMinor, 68 | repoVersionMajor, 69 | }; 70 | }; 71 | 72 | async init(githubPrivateKey: string, githubClientSecret: string): Promise { 73 | this._gitHub = await GitHub.init(githubPrivateKey, githubClientSecret); 74 | 75 | return this; 76 | } 77 | 78 | async ensureThatBaseImageHasBeenBuilt(): Promise { 79 | // Get base image information 80 | const jobId = CiJobs.parseJobId(Image.types.base, this.repoVersion); 81 | const job = await CiJobs.get(jobId); 82 | if (job === null) { 83 | throw new Error('[Scheduler] Expected base job to be present'); 84 | } 85 | 86 | // Schedule it 87 | if (['created', 'failed'].includes(job.status)) { 88 | const { repoVersionFull, repoVersionMinor, repoVersionMajor } = this; 89 | const response = await this.gitHub.repos.createDispatchEvent({ 90 | owner: 'unity-ci', 91 | repo: 'docker', 92 | event_type: GitHubWorkflow.eventTypes.newBaseImages, 93 | client_payload: { 94 | jobId, 95 | repoVersionFull, 96 | repoVersionMinor, 97 | repoVersionMajor, 98 | }, 99 | }); 100 | 101 | if (response.status <= 199 || response.status >= 300) { 102 | const failureMessage = ` 103 | [Scheduler] failed to schedule job ${jobId}, 104 | status: ${response.status}, response: ${response.data}`; 105 | logger.error(failureMessage); 106 | await Discord.sendAlert(failureMessage); 107 | return false; 108 | } 109 | 110 | await CiJobs.markJobAsScheduled(jobId); 111 | await Discord.sendDebug(`[Scheduler] Scheduled new base image build (${jobId}).`); 112 | return false; 113 | } 114 | 115 | // Don't do anything before base image is completed 116 | return job.status === 'completed'; 117 | } 118 | 119 | async ensureThatHubImageHasBeenBuilt(): Promise { 120 | // Get hub image information 121 | const jobId = CiJobs.parseJobId(Image.types.hub, this.repoVersion); 122 | const job = await CiJobs.get(jobId); 123 | if (job === null) { 124 | throw new Error('[Scheduler] Expected hub job to be present'); 125 | } 126 | 127 | // Schedule it 128 | if (['created', 'failed'].includes(job.status)) { 129 | const { repoVersionFull, repoVersionMinor, repoVersionMajor } = this; 130 | const response = await this.gitHub.repos.createDispatchEvent({ 131 | owner: 'unity-ci', 132 | repo: 'docker', 133 | event_type: GitHubWorkflow.eventTypes.newHubImages, 134 | client_payload: { 135 | jobId, 136 | repoVersionFull, 137 | repoVersionMinor, 138 | repoVersionMajor, 139 | }, 140 | }); 141 | 142 | if (response.status <= 199 || response.status >= 300) { 143 | const failureMessage = ` 144 | [Scheduler] failed to schedule job ${jobId}, 145 | status: ${response.status}, response: ${response.data}`; 146 | logger.error(failureMessage); 147 | await Discord.sendAlert(failureMessage); 148 | return false; 149 | } 150 | 151 | await CiJobs.markJobAsScheduled(jobId); 152 | await Discord.sendDebug(`[Scheduler] Scheduled new hub image build (${jobId})`); 153 | return false; 154 | } 155 | 156 | // Don't do anything before hub image is completed 157 | return job.status === 'completed'; 158 | } 159 | 160 | /** 161 | * Note: this is an important check 162 | * CiBuilds will go back to status "in progress", whereas 163 | * CiJobs will stay "failed" until all builds complete. 164 | * This will prevent creating failures on 1000+ builds 165 | */ 166 | async ensureThereAreNoFailedJobs(): Promise { 167 | const { maxToleratedFailures, maxExtraJobsForRescheduling } = settings; 168 | const failingJobs = await CiJobs.getFailingJobsQueue(); 169 | 170 | if (failingJobs.length >= 1) { 171 | const openSpots = await this.determineOpenSpots(); 172 | const numberToReschedule = openSpots + maxExtraJobsForRescheduling; 173 | 174 | if (numberToReschedule <= 0) { 175 | await Discord.sendDebug('[Scheduler] Not retrying any new jobs, as the queue is full'); 176 | return false; 177 | } 178 | 179 | const ingeminator = new Ingeminator(numberToReschedule, this.gitHub, this.repoVersionInfo); 180 | await ingeminator.rescheduleFailedJobs(failingJobs); 181 | } 182 | 183 | return failingJobs.length <= maxToleratedFailures; 184 | } 185 | 186 | async buildLatestEditorImages(): Promise { 187 | const openSpots = await this.determineOpenSpots(); 188 | if (openSpots <= 0) { 189 | await Discord.sendDebug('[Scheduler] Not scheduling any new jobs, as the queue is full'); 190 | return false; 191 | } 192 | 193 | // Repo version 194 | const { repoVersionFull, repoVersionMinor, repoVersionMajor } = this; 195 | 196 | // Get highest priority builds 197 | const queue = await CiJobs.getPrioritisedQueue(); 198 | 199 | // If the queue has nothing to build, we're happy 200 | if (queue.length <= 0) return true; 201 | 202 | // Schedule CiJobs as workflows, which will report back CiBuilds. 203 | const toBeScheduledJobs = take(queue, openSpots); 204 | const jobsAsString = toBeScheduledJobs?.map((job) => job.id).join(',\n'); 205 | await Discord.sendDebug(`[Scheduler] top of the queue: \n ${jobsAsString}`); 206 | for (const toBeScheduledJob of toBeScheduledJobs) { 207 | const { id: jobId, data } = toBeScheduledJob; 208 | 209 | const editorVersionInfo = data.editorVersionInfo as EditorVersionInfo; 210 | const { version: editorVersion, changeSet } = editorVersionInfo; 211 | const eventType = GitHubWorkflow.getEventTypeForEditorCiJob(editorVersionInfo); 212 | 213 | const response = await this.gitHub.repos.createDispatchEvent({ 214 | owner: 'unity-ci', 215 | repo: 'docker', 216 | event_type: eventType, 217 | client_payload: { 218 | jobId, 219 | editorVersion, 220 | changeSet, 221 | repoVersionFull, 222 | repoVersionMinor, 223 | repoVersionMajor, 224 | }, 225 | }); 226 | 227 | if (response.status <= 199 || response.status >= 300) { 228 | const failureMessage = ` 229 | [Scheduler] failed to schedule job ${jobId}, 230 | status: ${response.status}, response: ${response.data}`; 231 | logger.error(failureMessage); 232 | await Discord.sendAlert(failureMessage); 233 | return false; 234 | } 235 | 236 | await CiJobs.markJobAsScheduled(jobId); 237 | await Discord.sendDebug(`[Scheduler] Scheduled new editor image build (${jobId}).`); 238 | } 239 | 240 | // The queue was not empty, so we're not happy yet 241 | return false; 242 | } 243 | 244 | private async determineOpenSpots(): Promise { 245 | const currentlyRunningJobs = await CiJobs.getNumberOfScheduledJobs(); 246 | const openSpots = this.maxConcurrentJobs - currentlyRunningJobs; 247 | return openSpots <= 0 ? 0 : openSpots; 248 | } 249 | } 250 | -------------------------------------------------------------------------------- /functions/src/logic/dataTransformation/dataMigrations.ts: -------------------------------------------------------------------------------- 1 | export const dataMigrations = async () => { 2 | return; 3 | }; 4 | -------------------------------------------------------------------------------- /functions/src/logic/dataTransformation/index.ts: -------------------------------------------------------------------------------- 1 | export { dataMigrations } from './dataMigrations'; 2 | -------------------------------------------------------------------------------- /functions/src/logic/ingestRepoVersions/index.ts: -------------------------------------------------------------------------------- 1 | import { logger } from 'firebase-functions/v2'; 2 | import { Discord } from '../../service/discord'; 3 | import { scrapeVersions } from './scrapeVersions'; 4 | import { updateDatabase } from './updateDatabase'; 5 | 6 | export const ingestRepoVersions = async (githubPrivateKey: string, githubClientSecret: string) => { 7 | try { 8 | const scrapedInfoList = await scrapeVersions(githubPrivateKey, githubClientSecret); 9 | 10 | // Note: this triggers repoVersionInfo.onCreate modelTrigger 11 | await updateDatabase(scrapedInfoList); 12 | } catch (err: any) { 13 | const message = ` 14 | Something went wrong while importing repository versions for unity-ci/docker: 15 | ${err.message} (${err.status})\n${err.stackTrace} 16 | `; 17 | 18 | logger.error(message); 19 | await Discord.sendAlert(message); 20 | } 21 | }; 22 | -------------------------------------------------------------------------------- /functions/src/logic/ingestRepoVersions/scrapeVersions.ts: -------------------------------------------------------------------------------- 1 | import semver from 'semver'; 2 | import { RepoVersionInfo } from '../../model/repoVersionInfo'; 3 | import { GitHub } from '../../service/github'; 4 | 5 | export const scrapeVersions = async ( 6 | githubPrivateKey: string, 7 | githubClientSecret: string, 8 | ): Promise => { 9 | const gitHub = await GitHub.init(githubPrivateKey, githubClientSecret); 10 | 11 | const releases = await gitHub.repos.listReleases({ 12 | owner: 'unity-ci', 13 | repo: 'docker', 14 | }); 15 | 16 | const versions = releases.data.map((release) => { 17 | const { 18 | id, 19 | url, 20 | name, 21 | body: description, 22 | tag_name: tagName, 23 | author: { login: author }, 24 | target_commitish: commitIsh, 25 | } = release; 26 | 27 | const version = semver.valid(semver.coerce(tagName)); 28 | if (!version) { 29 | throw new Error("Assumed versions to always be parsable, but they're not."); 30 | } 31 | const major = semver.major(version); 32 | const minor = semver.minor(version); 33 | const patch = semver.patch(version); 34 | 35 | return { 36 | version, 37 | major, 38 | minor, 39 | patch, 40 | id, 41 | name, 42 | description, 43 | author, 44 | commitIsh, 45 | url, 46 | } as RepoVersionInfo; 47 | }); 48 | 49 | return versions; 50 | }; 51 | -------------------------------------------------------------------------------- /functions/src/logic/ingestRepoVersions/updateDatabase.ts: -------------------------------------------------------------------------------- 1 | import { isMatch } from 'lodash'; 2 | import { RepoVersionInfo } from '../../model/repoVersionInfo'; 3 | import { logger } from 'firebase-functions/v2'; 4 | import { Discord } from '../../service/discord'; 5 | 6 | const plural = (amount: number) => { 7 | return amount === 1 ? 'version' : 'versions'; 8 | }; 9 | 10 | export const updateDatabase = async (ingestedInfoList: RepoVersionInfo[]): Promise => { 11 | const existingInfoList = await RepoVersionInfo.getAll(); 12 | 13 | const newVersions: RepoVersionInfo[] = []; 14 | const updatedVersions: RepoVersionInfo[] = []; 15 | 16 | ingestedInfoList.forEach((ingestedInfo: RepoVersionInfo) => { 17 | const { version } = ingestedInfo; 18 | const existingVersion = existingInfoList.find((info) => info.version === version); 19 | 20 | if (!existingVersion) { 21 | newVersions.push(ingestedInfo); 22 | return; 23 | } 24 | 25 | if (!isMatch(existingVersion, ingestedInfo)) { 26 | updatedVersions.push(ingestedInfo); 27 | return; 28 | } 29 | }); 30 | 31 | let message = ''; 32 | 33 | if (newVersions.length >= 1) { 34 | await RepoVersionInfo.createMany(newVersions); 35 | const pluralNew = newVersions.length === 1 ? 'New' : `${newVersions.length} new`; 36 | message += ` 37 | ${pluralNew} repository ${plural(newVersions.length)} detected. 38 | (\`${newVersions.map((version) => version.version).join('`, `')}\`)`; 39 | } 40 | 41 | if (updatedVersions.length >= 1) { 42 | await RepoVersionInfo.updateMany(updatedVersions); 43 | const pluralUpdated = 44 | updatedVersions.length === 1 ? 'Updated' : `${updatedVersions.length} updated`; 45 | message += ` 46 | ${pluralUpdated} repository ${plural(updatedVersions.length)} detected. 47 | (\`${updatedVersions.map((version) => version.version).join('`, `')}\`)`; 48 | } 49 | 50 | message = message.trimEnd(); 51 | if (message.length >= 1) { 52 | logger.info(message); 53 | await Discord.sendNews(message); 54 | } else { 55 | logger.info('Database is up-to-date. (no updated repo versions found)'); 56 | } 57 | }; 58 | -------------------------------------------------------------------------------- /functions/src/logic/ingestUnityVersions/index.ts: -------------------------------------------------------------------------------- 1 | import { updateDatabase } from './updateDatabase'; 2 | import { logger } from 'firebase-functions/v2'; 3 | import { Discord } from '../../service/discord'; 4 | import { scrapeVersions } from './scrapeVersions'; 5 | 6 | export const ingestUnityVersions = async () => { 7 | try { 8 | const scrapedInfoList = await scrapeVersions(); 9 | 10 | // Note: this triggers editorVersionInfo.onCreate modelTrigger 11 | await updateDatabase(scrapedInfoList); 12 | } catch (err: any) { 13 | const message = ` 14 | Something went wrong while importing new versions from unity: 15 | ${err.message} (${err.status})\n${err.stackTrace} 16 | `; 17 | 18 | logger.error(message); 19 | await Discord.sendAlert(message); 20 | } 21 | }; 22 | -------------------------------------------------------------------------------- /functions/src/logic/ingestUnityVersions/scrapeVersions.ts: -------------------------------------------------------------------------------- 1 | import { EditorVersionInfo } from '../../model/editorVersionInfo'; 2 | import { searchChangesets, SearchMode } from 'unity-changeset'; 3 | 4 | const unity_version_regex = /^(\d+)\.(\d+)\.(\d+)([a-zA-Z]+)(-?\d+)$/; 5 | 6 | export const scrapeVersions = async (): Promise => { 7 | const unityVersions = await searchChangesets(SearchMode.Default); 8 | 9 | if (unityVersions?.length > 0) { 10 | return unityVersions 11 | .map((unityVersion) => { 12 | const match = RegExp(unity_version_regex).exec(unityVersion.version); 13 | if (match) { 14 | const [_, major, minor, patch, lifecycle, build] = match; 15 | 16 | if (lifecycle !== 'f' || Number(major) < 2017) { 17 | return null; 18 | } 19 | 20 | return { 21 | version: unityVersion.version, 22 | changeSet: unityVersion.changeset, 23 | major: Number(major), 24 | minor: Number(minor), 25 | patch, 26 | } as EditorVersionInfo; 27 | } 28 | return null; 29 | }) 30 | .filter((versionInfo): versionInfo is EditorVersionInfo => versionInfo !== null); 31 | } 32 | 33 | throw new Error('No Unity versions found!'); 34 | }; 35 | -------------------------------------------------------------------------------- /functions/src/logic/ingestUnityVersions/updateDatabase.ts: -------------------------------------------------------------------------------- 1 | import { isMatch } from 'lodash'; 2 | import { logger } from 'firebase-functions/v2'; 3 | import { Discord } from '../../service/discord'; 4 | import { EditorVersionInfo } from '../../model/editorVersionInfo'; 5 | 6 | const plural = (amount: number) => { 7 | return amount === 1 ? 'version' : 'versions'; 8 | }; 9 | 10 | export const updateDatabase = async (ingestedInfoList: EditorVersionInfo[]): Promise => { 11 | const existingInfoList = await EditorVersionInfo.getAll(); 12 | 13 | const newVersions: EditorVersionInfo[] = []; 14 | const updatedVersions: EditorVersionInfo[] = []; 15 | 16 | ingestedInfoList.forEach((scrapedInfo) => { 17 | const { version } = scrapedInfo; 18 | const existingVersion = existingInfoList.find((info) => info.version === version); 19 | 20 | if (!existingVersion) { 21 | newVersions.push(scrapedInfo); 22 | return; 23 | } 24 | 25 | if (!isMatch(existingVersion, scrapedInfo)) { 26 | updatedVersions.push(scrapedInfo); 27 | return; 28 | } 29 | }); 30 | 31 | let message = ''; 32 | 33 | if (newVersions.length >= 1) { 34 | await EditorVersionInfo.createMany(newVersions); 35 | message += ` 36 | ${newVersions.length} new Unity editor ${plural(newVersions.length)} detected. 37 | (\`${newVersions.map((version) => version.version).join('`, `')}\`)`; 38 | } 39 | 40 | if (updatedVersions.length >= 1) { 41 | await EditorVersionInfo.updateMany(updatedVersions); 42 | message += ` 43 | ${updatedVersions.length} updated Unity editor ${plural(updatedVersions.length)} detected. 44 | (\`${updatedVersions.map((version) => version.version).join('`, `')}\`)`; 45 | } 46 | 47 | message = message.trimEnd(); 48 | if (message.length >= 1) { 49 | logger.info(message); 50 | await Discord.sendMessageToMaintainers(message); 51 | } else { 52 | await Discord.sendDebug('Database is up-to-date. (no updated Unity versions found)'); 53 | } 54 | }; 55 | -------------------------------------------------------------------------------- /functions/src/logic/init/assignDefaultAdmins.ts: -------------------------------------------------------------------------------- 1 | import admin from 'firebase-admin'; 2 | import { auth } from '../../service/firebase'; 3 | 4 | const getUserByEmailAddress = async (emailAddress: string): Promise => { 5 | const user = await auth.getUserByEmail(emailAddress); 6 | if (!user) throw new Error(`No user for ${emailAddress}, skipping.`); 7 | 8 | return user; 9 | }; 10 | 11 | const makeUserAnAdmin = async (user: admin.auth.UserRecord): Promise => { 12 | const { customClaims = {}, displayName } = user; 13 | if (customClaims.admin === true) { 14 | return; 15 | } 16 | 17 | const updatedClaims = { ...customClaims, admin: true }; 18 | await auth.setCustomUserClaims(user.uid, updatedClaims); 19 | console.log(`${displayName} is now an admin. Claims:`, updatedClaims); 20 | }; 21 | 22 | const makeAdminByEmailAddress = async (emailAddress: string): Promise => { 23 | try { 24 | const user = await getUserByEmailAddress(emailAddress); 25 | 26 | await makeUserAnAdmin(user); 27 | } catch (error: any) { 28 | console.log(`${emailAddress}: ${error.message}.`); 29 | } 30 | }; 31 | 32 | export const assignDefaultAdmins = async (adminEmailAddresses: string[]): Promise => { 33 | if (!adminEmailAddresses) { 34 | return; 35 | } 36 | 37 | for (const emailAddress of adminEmailAddresses) { 38 | await makeAdminByEmailAddress(emailAddress); 39 | } 40 | }; 41 | -------------------------------------------------------------------------------- /functions/src/model-triggers/editorVersionInfo.ts: -------------------------------------------------------------------------------- 1 | import { 2 | FirestoreEvent, 3 | onDocumentCreated, 4 | QueryDocumentSnapshot, 5 | } from 'firebase-functions/v2/firestore'; 6 | import { EditorVersionInfo } from '../model/editorVersionInfo'; 7 | import { CiJobs } from '../model/ciJobs'; 8 | import { RepoVersionInfo } from '../model/repoVersionInfo'; 9 | import { Discord } from '../service/discord'; 10 | import { Image } from '../model/image'; 11 | import { logger } from 'firebase-functions/v2'; 12 | import { defineSecret } from 'firebase-functions/params'; 13 | 14 | const discordToken = defineSecret('DISCORD_TOKEN'); 15 | 16 | export const onCreate = onDocumentCreated( 17 | { 18 | document: `${EditorVersionInfo.collection}/{itemId}`, 19 | secrets: [discordToken], 20 | }, 21 | async (snapshot: FirestoreEvent) => { 22 | await Discord.init(discordToken.value()); 23 | 24 | const editorVersionInfo = snapshot.data?.data() as EditorVersionInfo; 25 | const repoVersionInfo = await RepoVersionInfo.getLatest(); 26 | 27 | const jobId = CiJobs.generateJobId(Image.types.editor, repoVersionInfo, editorVersionInfo); 28 | if (await CiJobs.exists(jobId)) { 29 | const message = `Skipped creating CiJob for new editorVersion (${editorVersionInfo.version}).`; 30 | logger.warn(message); 31 | await Discord.sendAlert(message); 32 | Discord.disconnect(); 33 | return; 34 | } 35 | 36 | await CiJobs.create(jobId, Image.types.editor, repoVersionInfo, editorVersionInfo); 37 | logger.info(`CiJob created: ${jobId}`); 38 | Discord.disconnect(); 39 | }, 40 | ); 41 | -------------------------------------------------------------------------------- /functions/src/model-triggers/index.ts: -------------------------------------------------------------------------------- 1 | export * as editorVersion from './editorVersionInfo'; 2 | export * as repoVersion from './repoVersionInfo'; 3 | -------------------------------------------------------------------------------- /functions/src/model-triggers/repoVersionInfo.ts: -------------------------------------------------------------------------------- 1 | import { 2 | FirestoreEvent, 3 | onDocumentCreated, 4 | QueryDocumentSnapshot, 5 | } from 'firebase-functions/v2/firestore'; 6 | import { db } from '../service/firebase'; 7 | import { logger } from 'firebase-functions/v2'; 8 | 9 | import { RepoVersionInfo } from '../model/repoVersionInfo'; 10 | import { CiJobs } from '../model/ciJobs'; 11 | import { EditorVersionInfo } from '../model/editorVersionInfo'; 12 | import semver from 'semver/preload'; 13 | import { Discord } from '../service/discord'; 14 | import { chunk } from 'lodash'; 15 | import { Image } from '../model/image'; 16 | import { defineSecret } from 'firebase-functions/params'; 17 | 18 | const discordToken = defineSecret('DISCORD_TOKEN'); 19 | 20 | export const onCreate = onDocumentCreated( 21 | { 22 | document: `${RepoVersionInfo.collection}/{itemId}`, 23 | secrets: [discordToken], 24 | }, 25 | async (snapshot: FirestoreEvent) => { 26 | await Discord.init(discordToken.value()); 27 | 28 | const repoVersionInfo = snapshot.data?.data() as RepoVersionInfo; 29 | const currentRepoVersion = repoVersionInfo.version; 30 | const latestRepoVersionInfo = await RepoVersionInfo.getLatest(); 31 | 32 | // Only create new builds for tags that are newer semantic versions. 33 | if (semver.compare(currentRepoVersion, latestRepoVersionInfo.version) !== 0) { 34 | const semverMessage = ` 35 | Skipped scheduling all editorVersions for new repoVersion, 36 | as it does not seem to be the newest version.`; 37 | logger.warn(semverMessage); 38 | await Discord.sendAlert(semverMessage); 39 | Discord.disconnect(); 40 | return; 41 | } 42 | 43 | // Skip creating jobs that already exist. 44 | const existingJobIds = await CiJobs.getAllIds(); 45 | const editorVersionInfos = await EditorVersionInfo.getAll(); 46 | const skippedVersions: string[] = []; 47 | 48 | // Create database batch transaction 49 | const baseAndHubBatch = db.batch(); 50 | 51 | // Job for base image 52 | const baseJobId = CiJobs.generateJobId('base', repoVersionInfo); 53 | if (existingJobIds.includes(baseJobId)) { 54 | skippedVersions.push(baseJobId); 55 | } else { 56 | const baseJobData = CiJobs.construct('base', repoVersionInfo); 57 | const baseJobRef = db.collection(CiJobs.collection).doc(baseJobId); 58 | baseAndHubBatch.create(baseJobRef, baseJobData); 59 | } 60 | 61 | // Job for hub image 62 | const hubJobId = CiJobs.generateJobId('hub', repoVersionInfo); 63 | if (existingJobIds.includes(hubJobId)) { 64 | skippedVersions.push(hubJobId); 65 | } else { 66 | const hubJobData = CiJobs.construct('hub', repoVersionInfo); 67 | const hubJobRef = db.collection(CiJobs.collection).doc(hubJobId); 68 | baseAndHubBatch.create(hubJobRef, hubJobData); 69 | } 70 | 71 | // End database bash transaction 72 | await baseAndHubBatch.commit(); 73 | 74 | // Batches can only have 20 document access calls per transaction 75 | // See: https://firebase.google.com/docs/firestore/manage-data/transactions 76 | // Note that batch.set uses 2 document access calls. 77 | // But now `create` seems to also be supported in batch calls 78 | const editorVersionInfoChunks: EditorVersionInfo[][] = chunk(editorVersionInfos, 20); 79 | for (const editorVersionInfoChunk of editorVersionInfoChunks) { 80 | const imageType = Image.types.editor; 81 | const batch = db.batch(); 82 | for (const editorVersionInfo of editorVersionInfoChunk) { 83 | const editorJobId = CiJobs.generateJobId(imageType, repoVersionInfo, editorVersionInfo); 84 | 85 | if (existingJobIds.includes(editorJobId)) { 86 | skippedVersions.push(editorJobId); 87 | } else { 88 | const editorJobData = CiJobs.construct(imageType, repoVersionInfo, editorVersionInfo); 89 | const editorJobRef = db.collection(CiJobs.collection).doc(editorJobId); 90 | batch.create(editorJobRef, editorJobData); 91 | } 92 | } 93 | await batch.commit(); 94 | } 95 | // Wow, haha 😅 96 | 97 | // Report skipped versions 98 | if (skippedVersions.length >= 1) { 99 | const skippedVersionsMessage = ` 100 | Skipped creating CiJobs for the following jobs \`${skippedVersions.join('`, `')}\`.`; 101 | logger.warn(skippedVersionsMessage); 102 | await Discord.sendAlert(skippedVersionsMessage); 103 | } 104 | 105 | // Report that probably many new jobs have now been scheduled 106 | const baseCount = 1; 107 | const hubCount = 1; 108 | const totalNewJobs = editorVersionInfos.length + baseCount + hubCount - skippedVersions.length; 109 | const newJobs = CiJobs.pluralise(totalNewJobs); 110 | const newJobsMessage = `Created ${newJobs} for version \`${currentRepoVersion}\` of unity-ci/docker.`; 111 | logger.info(newJobsMessage); 112 | await Discord.sendNews(newJobsMessage); 113 | 114 | // Supersede any non-complete jobs before the current version 115 | const numSuperseded = await CiJobs.markJobsBeforeRepoVersionAsSuperseded(currentRepoVersion); 116 | if (numSuperseded >= 1) { 117 | const replacementMessage = ` 118 | ${CiJobs.pluralise(numSuperseded)} that were for older versions are now superseded.`; 119 | logger.warn(replacementMessage); 120 | await Discord.sendMessageToMaintainers(replacementMessage); 121 | } else { 122 | logger.debug('no versions were superseded, as expected.'); 123 | } 124 | 125 | await Discord.disconnect(); 126 | }, 127 | ); 128 | -------------------------------------------------------------------------------- /functions/src/model/ciBuilds.ts: -------------------------------------------------------------------------------- 1 | import { admin, db } from '../service/firebase'; 2 | import { logger } from 'firebase-functions/v2'; 3 | import { settings } from '../config/settings'; 4 | import { CiJobs } from './ciJobs'; 5 | import { BaseOs, ImageType } from './image'; 6 | import Timestamp = admin.firestore.Timestamp; 7 | import FieldValue = admin.firestore.FieldValue; 8 | 9 | export type BuildStatus = 'started' | 'failed' | 'published'; 10 | 11 | // Used in Start API 12 | export interface BuildInfo { 13 | baseOs: BaseOs; 14 | repoVersion: string; 15 | editorVersion: string; 16 | targetPlatform: string; 17 | } 18 | 19 | // Used in Failure API 20 | export interface BuildFailure { 21 | reason: string; 22 | } 23 | 24 | // Used in Publish API 25 | export interface DockerInfo { 26 | imageRepo: string; 27 | imageName: string; 28 | friendlyTag: string; 29 | specificTag: string; 30 | digest: string; 31 | // date with docker as source of truth? 32 | } 33 | 34 | interface MetaData { 35 | lastBuildStart: Timestamp | null; 36 | failureCount: number; 37 | lastBuildFailure: Timestamp | null; 38 | publishedDate: Timestamp | null; 39 | } 40 | 41 | export interface CiBuild { 42 | buildId: string; 43 | relatedJobId: string; 44 | status: BuildStatus; 45 | imageType: ImageType; 46 | meta: MetaData; 47 | buildInfo: BuildInfo; 48 | failure: BuildFailure | null; 49 | dockerInfo: DockerInfo | null; 50 | addedDate: Timestamp; 51 | modifiedDate: Timestamp; 52 | } 53 | 54 | export type CiBuildQueueItem = { id: string; data: CiBuild }; 55 | export type CiBuildQueue = CiBuildQueueItem[]; 56 | 57 | /** 58 | * A CI Build represents a single [baseOs-unityVersion-targetPlatform] build. 59 | * These builds are reported in and run on GitHub Actions. 60 | * Statuses (failures and publications) are also reported back on this level. 61 | */ 62 | export class CiBuilds { 63 | public static get collection() { 64 | return 'ciBuilds'; 65 | } 66 | 67 | public static getAll = async (): Promise => { 68 | const snapshot = await db.collection(CiBuilds.collection).get(); 69 | 70 | return snapshot.docs.map((doc) => doc.data()) as CiBuild[]; 71 | }; 72 | 73 | public static get = async (buildId: string): Promise => { 74 | const snapshot = await db.doc(`${CiBuilds.collection}/${buildId}`).get(); 75 | 76 | if (!snapshot.exists) { 77 | return null; 78 | } 79 | 80 | return snapshot.data() as CiBuild; 81 | }; 82 | 83 | public static getStartedBuilds = async (): Promise => { 84 | const realisticMaximumConcurrentBuilds = settings.maxConcurrentJobs * 10; 85 | 86 | const snapshot = await db 87 | .collection(CiBuilds.collection) 88 | .where('status', '==', 'started') 89 | .limit(realisticMaximumConcurrentBuilds) 90 | .get(); 91 | 92 | return snapshot.docs.map((doc) => doc.data() as CiBuild); 93 | }; 94 | 95 | public static getFailedBuildsQueue = async (jobId: string): Promise => { 96 | const snapshot = await db 97 | .collection(CiBuilds.collection) 98 | .where('relatedJobId', '==', jobId) 99 | .where('status', '==', 'failed') 100 | .limit(settings.maxConcurrentJobs) 101 | .get(); 102 | 103 | return snapshot.docs.map((doc) => ({ 104 | id: doc.id, 105 | data: doc.data() as CiBuild, 106 | })); 107 | }; 108 | 109 | public static registerNewBuild = async ( 110 | buildId: string, 111 | relatedJobId: string, 112 | imageType: ImageType, 113 | buildInfo: BuildInfo, 114 | ) => { 115 | const data: CiBuild = { 116 | status: 'started', 117 | buildId, 118 | relatedJobId, 119 | imageType, 120 | buildInfo, 121 | failure: null, 122 | dockerInfo: null, 123 | meta: { 124 | lastBuildStart: Timestamp.now(), 125 | failureCount: 0, 126 | lastBuildFailure: null, 127 | publishedDate: null, 128 | }, 129 | addedDate: Timestamp.now(), 130 | modifiedDate: Timestamp.now(), 131 | }; 132 | 133 | const ref = await db.collection(CiBuilds.collection).doc(buildId); 134 | const snapshot = await ref.get(); 135 | 136 | let result; 137 | if (snapshot.exists) { 138 | // Builds can be retried after a failure. 139 | if (snapshot.data()?.status === 'failed') { 140 | // In case or reporting a new build during retry step, only overwrite these fields 141 | result = await ref.set(data, { 142 | mergeFields: ['status', 'meta.lastBuildStart', 'modifiedDate'], 143 | }); 144 | } else { 145 | throw new Error(`A build with "${buildId}" as identifier already exists`); 146 | } 147 | } else { 148 | result = await ref.create(data); 149 | } 150 | 151 | logger.debug('Build created', result); 152 | }; 153 | 154 | public static async removeDryRunBuild(buildId: string) { 155 | if (!buildId.startsWith('dryRun')) { 156 | throw new Error('Unexpected behaviour, expected only dryRun builds to be deleted'); 157 | } 158 | 159 | const ref = await db.collection(CiBuilds.collection).doc(buildId); 160 | const doc = await ref.get(); 161 | logger.info('dryRun produced this build endResult', doc.data()); 162 | 163 | await ref.delete(); 164 | } 165 | 166 | public static markBuildAsFailed = async (buildId: string, failure: BuildFailure) => { 167 | const build = await db.collection(CiBuilds.collection).doc(buildId); 168 | 169 | await build.update({ 170 | status: 'failed', 171 | failure, 172 | modifiedDate: Timestamp.now(), 173 | 'meta.failureCount': FieldValue.increment(1), 174 | 'meta.lastBuildFailure': Timestamp.now(), 175 | }); 176 | }; 177 | 178 | public static markBuildAsPublished = async ( 179 | buildId: string, 180 | jobId: string, 181 | dockerInfo: DockerInfo, 182 | ): Promise => { 183 | const build = await db.collection(CiBuilds.collection).doc(buildId); 184 | 185 | await build.update({ 186 | status: 'published', 187 | dockerInfo, 188 | modifiedDate: Timestamp.now(), 189 | 'meta.publishedDate': Timestamp.now(), 190 | }); 191 | 192 | const parentJobIsNowCompleted = await CiBuilds.haveAllBuildsForJobBeenPublished(jobId); 193 | if (parentJobIsNowCompleted) { 194 | await CiJobs.markJobAsCompleted(jobId); 195 | } 196 | 197 | return parentJobIsNowCompleted; 198 | }; 199 | 200 | public static haveAllBuildsForJobBeenPublished = async (jobId: string): Promise => { 201 | const snapshot = await db 202 | .collection(CiBuilds.collection) 203 | .where('relatedJobId', '==', jobId) 204 | .where('status', '!=', 'published') 205 | .limit(1) 206 | .get(); 207 | 208 | return snapshot.docs.length === 0; 209 | }; 210 | } 211 | -------------------------------------------------------------------------------- /functions/src/model/ciJobs.ts: -------------------------------------------------------------------------------- 1 | import { admin, db } from '../service/firebase'; 2 | import { EditorVersionInfo } from './editorVersionInfo'; 3 | import FieldValue = admin.firestore.FieldValue; 4 | import Timestamp = admin.firestore.Timestamp; 5 | import { RepoVersionInfo } from './repoVersionInfo'; 6 | import DocumentSnapshot = admin.firestore.DocumentSnapshot; 7 | import { chunk } from 'lodash'; 8 | import { settings } from '../config/settings'; 9 | import { Image, ImageType } from './image'; 10 | import { logger } from 'firebase-functions/v2'; 11 | 12 | export type JobStatus = 13 | | 'created' 14 | | 'scheduled' 15 | | 'inProgress' 16 | | 'completed' 17 | | 'failed' 18 | | 'superseded' 19 | | 'deprecated'; 20 | 21 | interface MetaData { 22 | lastBuildStart: Timestamp | null; 23 | failureCount: number; 24 | lastBuildFailure: Timestamp | null; 25 | } 26 | 27 | export interface CiJob { 28 | status: JobStatus; 29 | meta: MetaData; 30 | imageType: ImageType; 31 | repoVersionInfo: RepoVersionInfo; 32 | editorVersionInfo: EditorVersionInfo | null; 33 | addedDate: Timestamp; 34 | modifiedDate: Timestamp; 35 | } 36 | export type CiJobQueueItem = { id: string; data: CiJob }; 37 | export type CiJobQueue = CiJobQueueItem[]; 38 | 39 | /** 40 | * A CI job is a high level job, that schedules builds on a [repoVersion-unityVersion] level 41 | */ 42 | export class CiJobs { 43 | public static get collection() { 44 | return 'ciJobs'; 45 | } 46 | 47 | static get = async (jobId: string): Promise => { 48 | const ref = await db.collection(CiJobs.collection).doc(jobId); 49 | const snapshot = await ref.get(); 50 | 51 | if (!snapshot.exists) { 52 | return null; 53 | } 54 | 55 | return snapshot.data() as CiJob; 56 | }; 57 | 58 | static exists = async (jobId: string): Promise => { 59 | return (await CiJobs.get(jobId)) !== null; 60 | }; 61 | 62 | static getAll = async (): Promise => { 63 | const snapshot = await db.collection(CiJobs.collection).get(); 64 | 65 | return snapshot.docs.map((doc) => doc.data()) as CiJob[]; 66 | }; 67 | 68 | static getAllIds = async (): Promise => { 69 | const snapshot = await db.collection(CiJobs.collection).get(); 70 | 71 | return snapshot.docs.map(({ id }) => id); 72 | }; 73 | 74 | static getPrioritisedQueue = async (): Promise => { 75 | // Note: we can't simply do select distinct major, max(minor), max(patch) in nosql 76 | const snapshot = await db 77 | .collection(CiJobs.collection) 78 | .orderBy('editorVersionInfo.major', 'desc') 79 | .orderBy('editorVersionInfo.minor', 'desc') 80 | .orderBy('editorVersionInfo.patch', 'desc') 81 | .where('status', '==', 'created') 82 | .limit(settings.maxConcurrentJobs) 83 | .get(); 84 | 85 | logger.debug(`BuildQueue size: ${snapshot.docs.length}`); 86 | 87 | const queue: CiJobQueue = []; 88 | snapshot.docs.forEach((doc) => { 89 | queue.push({ id: doc.id, data: doc.data() as CiJob }); 90 | }); 91 | 92 | logger.debug(`BuildQueue`, queue); 93 | 94 | return queue; 95 | }; 96 | 97 | static getFailingJobsQueue = async (): Promise => { 98 | const snapshot = await db 99 | .collection(CiJobs.collection) 100 | .orderBy('editorVersionInfo.major', 'desc') 101 | .orderBy('editorVersionInfo.minor', 'desc') 102 | .orderBy('editorVersionInfo.patch', 'desc') 103 | .where('status', '==', 'failed') 104 | .limit(settings.maxConcurrentJobs) 105 | .get(); 106 | 107 | logger.debug(`FailingQueue size: ${snapshot.docs.length}`); 108 | 109 | const queue: CiJobQueue = []; 110 | snapshot.docs.forEach((doc) => { 111 | queue.push({ id: doc.id, data: doc.data() as CiJob }); 112 | }); 113 | 114 | logger.debug(`FailingQueue`, queue); 115 | 116 | return queue; 117 | }; 118 | 119 | static getNumberOfScheduledJobs = async (): Promise => { 120 | const snapshot = await db 121 | .collection(CiJobs.collection) 122 | .where('status', 'in', ['scheduled', 'inProgress']) 123 | .limit(settings.maxConcurrentJobs) 124 | .get(); 125 | 126 | return snapshot.docs.length; 127 | }; 128 | 129 | static create = async ( 130 | jobId: string, 131 | imageType: ImageType, 132 | repoVersionInfo: RepoVersionInfo, 133 | editorVersionInfo: EditorVersionInfo | null = null, 134 | ) => { 135 | const job = CiJobs.construct(imageType, repoVersionInfo, editorVersionInfo); 136 | const result = await db.collection(CiJobs.collection).doc(jobId).create(job); 137 | logger.debug('Job created', result); 138 | }; 139 | 140 | static construct = ( 141 | imageType: ImageType, 142 | repoVersionInfo: RepoVersionInfo, 143 | editorVersionInfo: EditorVersionInfo | null = null, 144 | ): CiJob => { 145 | let status: JobStatus = 'deprecated'; 146 | if ( 147 | editorVersionInfo === null || 148 | editorVersionInfo.major >= 2019 || 149 | (editorVersionInfo.major === 2018 && editorVersionInfo.minor >= 2) 150 | ) { 151 | status = 'created'; 152 | } 153 | 154 | const job: CiJob = { 155 | status, 156 | imageType, 157 | repoVersionInfo, 158 | editorVersionInfo, 159 | meta: { 160 | lastBuildStart: null, 161 | failureCount: 0, 162 | lastBuildFailure: null, 163 | }, 164 | addedDate: Timestamp.now(), 165 | modifiedDate: Timestamp.now(), 166 | }; 167 | 168 | return job; 169 | }; 170 | 171 | static markJobAsScheduled = async (jobId: string) => { 172 | const ref = await db.collection(CiJobs.collection).doc(jobId); 173 | const snapshot = await ref.get(); 174 | 175 | if (!snapshot.exists) { 176 | throw new Error(`Trying to mark job '${jobId}' as scheduled. But it does not exist.`); 177 | } 178 | 179 | const currentBuild = snapshot.data() as CiJob; 180 | 181 | // Do not override failure or completed 182 | // In CiJobs, "failure" is used to not race past failed jobs in the buildQueue, whereas 183 | // in CiBuilds the status may be marked as "inProgress" when retrying. 184 | let { status } = currentBuild; 185 | if (['created'].includes(status)) { 186 | status = 'scheduled'; 187 | } 188 | 189 | await ref.update({ 190 | status, 191 | modifiedDate: Timestamp.now(), 192 | }); 193 | }; 194 | 195 | static markJobAsInProgress = async (jobId: string) => { 196 | const ref = await db.collection(CiJobs.collection).doc(jobId); 197 | const snapshot = await ref.get(); 198 | 199 | if (!snapshot.exists) { 200 | throw new Error(`Trying to mark job '${jobId}' as in progress. But it does not exist.`); 201 | } 202 | 203 | const currentBuild = snapshot.data() as CiJob; 204 | logger.warn(currentBuild); 205 | 206 | // Do not override failure or completed 207 | let { status } = currentBuild; 208 | if (['scheduled'].includes(status)) { 209 | status = 'inProgress'; 210 | } 211 | 212 | await ref.update({ 213 | status, 214 | 'meta.lastBuildStart': Timestamp.now(), 215 | modifiedDate: Timestamp.now(), 216 | }); 217 | }; 218 | 219 | static markFailureForJob = async (jobId: string) => { 220 | const job = await db.collection(CiJobs.collection).doc(jobId); 221 | 222 | await job.update({ 223 | status: 'failed', 224 | 'meta.failureCount': FieldValue.increment(1), 225 | 'meta.lastBuildFailure': Timestamp.now(), 226 | modifiedDate: Timestamp.now(), 227 | }); 228 | }; 229 | 230 | static markJobAsCompleted = async (jobId: string) => { 231 | const job = await db.collection(CiJobs.collection).doc(jobId); 232 | 233 | await job.update({ 234 | status: 'completed', 235 | modifiedDate: Timestamp.now(), 236 | }); 237 | }; 238 | 239 | static async removeDryRunJob(jobId: string) { 240 | if (!jobId.startsWith('dryRun')) { 241 | throw new Error('Expect only dryRun jobs to be deleted.'); 242 | } 243 | 244 | await db.collection(CiJobs.collection).doc(jobId).delete(); 245 | } 246 | 247 | static markJobsBeforeRepoVersionAsSuperseded = async (repoVersion: string): Promise => { 248 | logger.info('superseding jobs before repo version', repoVersion); 249 | 250 | let numSuperseded = 0; 251 | for (const state of ['created', 'failed']) { 252 | // Note: Cannot have inequality filters on multiple properties (hence the forOf) 253 | const snapshot = await db 254 | .collection(CiJobs.collection) 255 | .where('repoVersionInfo.version', '<', repoVersion) 256 | .where('status', '==', state) 257 | .get(); 258 | 259 | numSuperseded += snapshot.docs.length; 260 | logger.debug(`superseding ${CiJobs.pluralise(numSuperseded)} with ${state} status`); 261 | 262 | // Batches can only have 20 document access calls per transaction 263 | // See: https://firebase.google.com/docs/firestore/manage-data/transactions 264 | // Note: Set counts as 2 access calls 265 | const status: JobStatus = 'superseded'; 266 | const docsChunks: DocumentSnapshot[][] = chunk(snapshot.docs, 10); 267 | for (const docsChunk of docsChunks) { 268 | const batch = db.batch(); 269 | for (const doc of docsChunk) { 270 | batch.set(doc.ref, { status }, { merge: true }); 271 | } 272 | await batch.commit(); 273 | logger.debug('committed batch of superseded jobs'); 274 | } 275 | } 276 | 277 | return numSuperseded; 278 | }; 279 | 280 | static generateJobId( 281 | imageType: ImageType, 282 | repoVersionInfo: RepoVersionInfo, 283 | editorVersionInfo: EditorVersionInfo | null = null, 284 | ) { 285 | const { version: repoVersion } = repoVersionInfo; 286 | if (imageType !== Image.types.editor) { 287 | return CiJobs.parseJobId(imageType, repoVersion); 288 | } 289 | 290 | if (editorVersionInfo === null) { 291 | throw new Error('editorVersionInfo must be provided for editor build jobs.'); 292 | } 293 | const { version: editorVersion } = editorVersionInfo; 294 | 295 | return CiJobs.parseJobId(imageType, repoVersion, editorVersion); 296 | } 297 | 298 | static parseJobId( 299 | imageType: ImageType, 300 | repoVersion: string, 301 | editorVersion: string | null = null, 302 | ) { 303 | if (imageType !== Image.types.editor) { 304 | return `${imageType}-${repoVersion}`; 305 | } 306 | 307 | if (editorVersion === null) { 308 | throw new Error('editorVersion must be provided for editor build jobs.'); 309 | } 310 | 311 | return `${imageType}-${editorVersion}-${repoVersion}`; 312 | } 313 | 314 | static pluralise(number: number) { 315 | const word = number === 1 ? 'CI Job' : 'CI Jobs'; 316 | return `${number} ${word}`; 317 | } 318 | } 319 | -------------------------------------------------------------------------------- /functions/src/model/ciVersionInfo.ts: -------------------------------------------------------------------------------- 1 | import { admin, db } from '../service/firebase'; 2 | import Timestamp = admin.firestore.Timestamp; 3 | import { EditorVersionInfo } from './editorVersionInfo'; 4 | import { RepoVersionInfo } from './repoVersionInfo'; 5 | import { logger } from 'firebase-functions/v2'; 6 | 7 | export interface CiVersionInfo { 8 | editorVersion: EditorVersionInfo; 9 | repoVersion: RepoVersionInfo; 10 | addedDate?: Timestamp; 11 | modifiedDate?: Timestamp; 12 | } 13 | 14 | export class CiVersionInfo { 15 | public static get collection() { 16 | return 'builtVersions'; 17 | } 18 | 19 | static getAll = async (): Promise => { 20 | const snapshot = await db 21 | .collection(CiVersionInfo.collection) 22 | .orderBy('editorVersion.major', 'desc') 23 | .orderBy('editorVersion.minor', 'desc') 24 | .orderBy('editorVersion.patch', 'desc') 25 | .orderBy('repoVersion.major', 'desc') 26 | .orderBy('repoVersion.minor', 'desc') 27 | .orderBy('repoVersion.patch', 'desc') 28 | .get(); 29 | 30 | return snapshot.docs.map((doc) => doc.data()) as CiVersionInfo[]; 31 | }; 32 | 33 | static create = async (editorVersion: EditorVersionInfo, repoVersion: RepoVersionInfo) => { 34 | try { 35 | await db.collection(CiVersionInfo.collection).doc('some elaborate id').set({ 36 | editorVersion, 37 | repoVersion, 38 | addedDate: Timestamp.now(), 39 | }); 40 | } catch (err) { 41 | logger.error('Error occurred during batch commit of new version', err); 42 | } 43 | }; 44 | } 45 | -------------------------------------------------------------------------------- /functions/src/model/editorVersionInfo.ts: -------------------------------------------------------------------------------- 1 | import { admin, db } from '../service/firebase'; 2 | import Timestamp = admin.firestore.Timestamp; 3 | import { logger } from 'firebase-functions/v2'; 4 | 5 | export const EDITOR_VERSIONS_COLLECTION = 'editorVersions'; 6 | 7 | export interface EditorVersionInfo { 8 | version: string; 9 | changeSet: string; 10 | major: number; 11 | minor: number; 12 | patch: string; 13 | addedDate?: Timestamp; 14 | modifiedDate?: Timestamp; 15 | } 16 | 17 | export class EditorVersionInfo { 18 | public static get collection() { 19 | return 'editorVersions'; 20 | } 21 | 22 | static get = async (version: string): Promise => { 23 | const snapshot = await db.collection(EditorVersionInfo.collection).doc(version).get(); 24 | 25 | return snapshot.data() as EditorVersionInfo; 26 | }; 27 | 28 | static getAllIds = async (): Promise => { 29 | const snapshot = await db 30 | .collection(EditorVersionInfo.collection) 31 | .orderBy('major', 'desc') 32 | .orderBy('minor', 'desc') 33 | .orderBy('patch', 'desc') 34 | .get(); 35 | 36 | return snapshot.docs.map((doc) => doc.id); 37 | }; 38 | 39 | static getAll = async (): Promise => { 40 | const snapshot = await db 41 | .collection(EditorVersionInfo.collection) 42 | .orderBy('major', 'desc') 43 | .orderBy('minor', 'desc') 44 | .orderBy('patch', 'desc') 45 | .get(); 46 | 47 | return snapshot.docs.map((doc) => doc.data()) as EditorVersionInfo[]; 48 | }; 49 | 50 | static createMany = async (editorVersionList: EditorVersionInfo[]) => { 51 | try { 52 | const batch = db.batch(); 53 | 54 | editorVersionList.forEach((versionInfo) => { 55 | const { version } = versionInfo; 56 | 57 | const ref = db.collection(EditorVersionInfo.collection).doc(version); 58 | const data = { 59 | ...versionInfo, 60 | addedDate: Timestamp.now(), 61 | modifiedDate: Timestamp.now(), 62 | }; 63 | batch.set(ref, data, { merge: false }); 64 | }); 65 | 66 | await batch.commit(); 67 | } catch (err) { 68 | logger.error('Error occurred during batch commit of new editor versions', err); 69 | } 70 | }; 71 | 72 | static updateMany = async (versionInfoList: EditorVersionInfo[]) => { 73 | try { 74 | const batch = db.batch(); 75 | 76 | versionInfoList.forEach((versionInfo) => { 77 | const { version } = versionInfo; 78 | 79 | const ref = db.collection(EditorVersionInfo.collection).doc(version); 80 | const data = { ...versionInfo, modifiedDate: Timestamp.now() }; 81 | batch.set(ref, data, { merge: true }); 82 | }); 83 | 84 | await batch.commit(); 85 | } catch (err) { 86 | logger.error('Error occurred during batch commit of new editor versions', err); 87 | } 88 | }; 89 | } 90 | -------------------------------------------------------------------------------- /functions/src/model/gitHubWorkflow.ts: -------------------------------------------------------------------------------- 1 | import { BaseOs } from './image'; 2 | import { EditorVersionInfo } from './editorVersionInfo'; 3 | 4 | export type GitHubEventType = 5 | | 'new_base_images_requested' 6 | | 'new_hub_images_requested' 7 | | 'new_legacy_editor_image_requested' 8 | | 'new_post_2019_2_editor_image_requested' 9 | | 'retry_ubuntu_editor_image_requested' 10 | | 'retry_windows_editor_image_requested'; 11 | 12 | export type FriendlyEventTypes = 13 | | 'newBaseImages' 14 | | 'newHubImages' 15 | | 'newLegacyImage' 16 | | 'newPost2019dot2Image' 17 | | 'retryUbuntuImage' 18 | | 'retryWindowsImage'; 19 | 20 | export class GitHubWorkflow { 21 | public static get eventTypes(): Record { 22 | return { 23 | newBaseImages: 'new_base_images_requested', 24 | newHubImages: 'new_hub_images_requested', 25 | newLegacyImage: 'new_legacy_editor_image_requested', 26 | newPost2019dot2Image: 'new_post_2019_2_editor_image_requested', 27 | retryUbuntuImage: 'retry_ubuntu_editor_image_requested', 28 | retryWindowsImage: 'retry_windows_editor_image_requested', 29 | }; 30 | } 31 | 32 | /** 33 | * Note: CiJob includes all builds for all base OSes 34 | */ 35 | public static getEventTypeForEditorCiJob(editorVersionInfo: EditorVersionInfo): GitHubEventType { 36 | const { major, minor } = editorVersionInfo; 37 | 38 | if (major >= 2020 || (major === 2019 && minor >= 3)) { 39 | return GitHubWorkflow.eventTypes.newPost2019dot2Image; 40 | } else { 41 | return GitHubWorkflow.eventTypes.newLegacyImage; 42 | } 43 | } 44 | 45 | /** 46 | * Note: CiBuild includes only a single baseOs-editorVersion-targetPlatform combination. 47 | */ 48 | public static getEventTypeForRetryingEditorCiBuild(baseOs: BaseOs): GitHubEventType { 49 | switch (baseOs) { 50 | case 'ubuntu': 51 | return GitHubWorkflow.eventTypes.retryUbuntuImage; 52 | case 'windows': 53 | return GitHubWorkflow.eventTypes.retryWindowsImage; 54 | default: 55 | throw new Error(`No retry method for base OS "${baseOs}" implemented.`); 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /functions/src/model/image.ts: -------------------------------------------------------------------------------- 1 | export type ImageType = 'base' | 'hub' | 'editor'; 2 | export type BaseOs = 'windows' | 'ubuntu'; 3 | export type TargetPlatform = 4 | | 'webgl' 5 | | 'mac-mono' 6 | | 'windows-mono' 7 | | 'windows-il2cpp' 8 | | 'universal-windows-platform' 9 | | 'base' 10 | | 'linux-il2cpp' 11 | | 'android' 12 | | 'ios' 13 | | 'appletv' 14 | | 'facebook'; 15 | 16 | export class Image { 17 | public static get types(): Record { 18 | return { 19 | base: 'base', 20 | hub: 'hub', 21 | editor: 'editor', 22 | }; 23 | } 24 | 25 | public static get baseOses(): Record { 26 | return { 27 | ubuntu: 'ubuntu', 28 | windows: 'windows', 29 | }; 30 | } 31 | 32 | public static get targetPlatformSuffixes(): Record { 33 | return { 34 | webgl: 'webgl', 35 | mac: 'mac-mono', 36 | windows: 'windows-mono', 37 | windowsIl2cpp: 'windows-il2cpp', 38 | wsaPlayer: 'universal-windows-platform', 39 | linux: 'base', 40 | linuxIl2cpp: 'linux-il2cpp', 41 | android: 'android', 42 | ios: 'ios', 43 | tvos: 'appletv', 44 | facebook: 'facebook', 45 | }; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /functions/src/model/repoVersionInfo.ts: -------------------------------------------------------------------------------- 1 | import { admin, db } from '../service/firebase'; 2 | import Timestamp = admin.firestore.Timestamp; 3 | import { logger } from 'firebase-functions/v2'; 4 | 5 | export interface RepoVersionInfo { 6 | id: number; 7 | version: string; 8 | major: number; 9 | minor: number; 10 | patch: number; 11 | name: string; 12 | description: string; 13 | author: string; 14 | url: string; 15 | commitIsh: string; 16 | addedDate?: Timestamp; 17 | modifiedDate?: Timestamp; 18 | } 19 | 20 | export class RepoVersionInfo { 21 | public static get collection() { 22 | return 'repoVersions'; 23 | } 24 | 25 | static getLatest = async (): Promise => { 26 | const snapshot = await db 27 | .collection(RepoVersionInfo.collection) 28 | .orderBy('major', 'desc') 29 | .orderBy('minor', 'desc') 30 | .orderBy('patch', 'desc') 31 | .limit(1) 32 | .get(); 33 | 34 | if (snapshot.docs.length <= 0) { 35 | throw new Error('No repository versions have been ingested yet'); 36 | } 37 | 38 | return snapshot.docs[0].data() as RepoVersionInfo; 39 | }; 40 | 41 | static getAllIds = async (): Promise => { 42 | const snapshot = await db 43 | .collection(RepoVersionInfo.collection) 44 | .orderBy('major', 'desc') 45 | .orderBy('minor', 'desc') 46 | .orderBy('patch', 'desc') 47 | .get(); 48 | 49 | return snapshot.docs.map((doc: any) => doc.id); 50 | }; 51 | 52 | static getAll = async (): Promise => { 53 | const snapshot = await db 54 | .collection(RepoVersionInfo.collection) 55 | .orderBy('major', 'desc') 56 | .orderBy('minor', 'desc') 57 | .orderBy('patch', 'desc') 58 | .get(); 59 | 60 | return snapshot.docs.map((doc: any) => doc.data()) as RepoVersionInfo[]; 61 | }; 62 | 63 | static create = async (repoVersion: RepoVersionInfo) => { 64 | const { version } = repoVersion; 65 | await db 66 | .collection(RepoVersionInfo.collection) 67 | .doc(version) 68 | .set( 69 | { 70 | ...repoVersion, 71 | addedDate: Timestamp.now(), 72 | modifiedDate: Timestamp.now(), 73 | }, 74 | { merge: false }, 75 | ); 76 | }; 77 | 78 | static update = async (repoVersion: RepoVersionInfo) => { 79 | const { version } = repoVersion; 80 | await db 81 | .collection(RepoVersionInfo.collection) 82 | .doc(version) 83 | .set( 84 | { 85 | ...repoVersion, 86 | modifiedDate: Timestamp.now(), 87 | }, 88 | { merge: true }, 89 | ); 90 | }; 91 | 92 | static createMany = async (repoVersionList: RepoVersionInfo[]) => { 93 | try { 94 | const batch = db.batch(); 95 | 96 | repoVersionList.forEach((versionInfo) => { 97 | const { version } = versionInfo; 98 | 99 | const ref = db.collection(RepoVersionInfo.collection).doc(version); 100 | const data = { 101 | ...versionInfo, 102 | addedDate: Timestamp.now(), 103 | modifiedDate: Timestamp.now(), 104 | }; 105 | batch.set(ref, data, { merge: false }); 106 | }); 107 | 108 | await batch.commit(); 109 | } catch (err) { 110 | logger.error('Error occurred during batch commit of new repo versions', err); 111 | } 112 | }; 113 | 114 | static updateMany = async (repoVersionList: RepoVersionInfo[]) => { 115 | try { 116 | const batch = db.batch(); 117 | 118 | repoVersionList.forEach((versionInfo) => { 119 | const { version } = versionInfo; 120 | 121 | const ref = db.collection(RepoVersionInfo.collection).doc(version); 122 | const data = { ...versionInfo, modifiedDate: Timestamp.now() }; 123 | batch.set(ref, data, { merge: true }); 124 | }); 125 | 126 | await batch.commit(); 127 | } catch (err) { 128 | logger.error('Error occurred during batch commit of new repo versions', err); 129 | } 130 | }; 131 | } 132 | -------------------------------------------------------------------------------- /functions/src/service/discord.ts: -------------------------------------------------------------------------------- 1 | import { Client as ErisClient, FileContent, Message, MessageContent } from 'eris'; 2 | import { logger } from 'firebase-functions/v2'; 3 | import { settings } from '../config/settings'; 4 | 5 | let instance: ErisClient | null = null; 6 | let instanceCount = 0; 7 | export class Discord { 8 | public static async init(token: string) { 9 | if (instance) { 10 | instanceCount += 1; 11 | return; 12 | } 13 | 14 | instance = new ErisClient(token); 15 | instance.on('messageCreate', async (message: Message) => { 16 | if (message.content === '!ping') { 17 | logger.info('[discord] pong!'); 18 | await message.channel.createMessage('Pong!'); 19 | } 20 | }); 21 | 22 | await instance.connect(); 23 | 24 | let secondsWaited = 0; 25 | while (!instance?.startTime) { 26 | await new Promise((resolve) => setTimeout(resolve, 1000)); 27 | secondsWaited += 1; 28 | 29 | if (secondsWaited >= 15) { 30 | throw new Error('Bot never became ready'); 31 | } 32 | } 33 | 34 | instanceCount += 1; 35 | } 36 | 37 | public static async sendDebugLine(message: 'begin' | 'end') { 38 | await instance?.createMessage(settings.discord.channels.debug, `--- ${message} ---`); 39 | } 40 | 41 | public static async sendDebug( 42 | message: MessageContent, 43 | files: FileContent | FileContent[] | undefined = undefined, 44 | ): Promise { 45 | logger.info(message); 46 | 47 | // Set to null as we don't differ between debug/info yet. Null is debug. 48 | return this.sendMessage(null, message, 'info', files); 49 | } 50 | 51 | public static async sendNews( 52 | message: MessageContent, 53 | files: FileContent | FileContent[] | undefined = undefined, 54 | ): Promise { 55 | return this.sendMessage(settings.discord.channels.news, message, 'info', files); 56 | } 57 | 58 | public static async sendAlert( 59 | message: MessageContent, 60 | files: FileContent | FileContent[] | undefined = undefined, 61 | ): Promise { 62 | return this.sendMessage(settings.discord.channels.alerts, message, 'error', files); 63 | } 64 | 65 | public static async sendMessageToMaintainers( 66 | message: MessageContent, 67 | files: FileContent | FileContent[] | undefined = undefined, 68 | ): Promise { 69 | return this.sendMessage(settings.discord.channels.maintainers, message, 'info', files); 70 | } 71 | 72 | private static async sendMessage( 73 | channelId: string | null, 74 | messageContent: MessageContent, 75 | level: 'debug' | 'info' | 'warn' | 'error' | 'critical', 76 | files: FileContent | FileContent[] | undefined = undefined, 77 | ): Promise { 78 | let isSent = false; 79 | try { 80 | if (typeof messageContent === 'string') { 81 | for (const message of Discord.splitMessage(messageContent)) { 82 | if (channelId) { 83 | await instance?.createMessage(channelId, message, files); 84 | } 85 | 86 | // Also send to debug channel 87 | await instance?.createMessage( 88 | settings.discord.channels.debug, 89 | `[${level}] ${message}`, 90 | files, 91 | ); 92 | } 93 | } else { 94 | if (channelId) { 95 | await instance?.createMessage(channelId, messageContent, files); 96 | } 97 | 98 | // Also send to debug channel 99 | messageContent.content = `[${level}] ${messageContent.content}`; 100 | await instance?.createMessage(settings.discord.channels.debug, messageContent, files); 101 | } 102 | 103 | isSent = true; 104 | } catch (err) { 105 | logger.error('An error occurred while trying to send a message to discord.', err); 106 | } 107 | 108 | return isSent; 109 | } 110 | 111 | public static disconnect() { 112 | if (!instance) return; 113 | instanceCount -= 1; 114 | 115 | if (instanceCount > 0) return; 116 | 117 | instance.disconnect({ reconnect: false }); 118 | instance = null; 119 | 120 | if (instanceCount < 0) { 121 | logger.error('Discord instance count is negative! This should not happen'); 122 | instanceCount = 0; 123 | } 124 | } 125 | 126 | // Max message size must account for ellipsis and level parts that are added to the message. 127 | static splitMessage(message: string, maxMessageSize: number = 1940): Array { 128 | const numberOfMessages = Math.ceil(message.length / maxMessageSize); 129 | const messages: Array = new Array(numberOfMessages); 130 | 131 | for (let i = 0, pointer = 0; i < numberOfMessages; i++) { 132 | let messageSize = maxMessageSize; 133 | 134 | let prefix = ''; 135 | if (i !== 0) { 136 | prefix = '...'; 137 | messageSize -= 3; 138 | } 139 | 140 | let suffix = ''; 141 | if (i !== numberOfMessages - 1) { 142 | suffix = '...'; 143 | messageSize -= 3; 144 | } 145 | 146 | // Break at spaces 147 | let maxMessage = message.slice(pointer, pointer + messageSize); 148 | const lastSpacePos = maxMessage.lastIndexOf(' '); 149 | if (lastSpacePos >= maxMessageSize - 250) { 150 | maxMessage = maxMessage.slice(pointer, pointer + lastSpacePos); 151 | messageSize = lastSpacePos; 152 | } 153 | 154 | messages[i] = `${prefix}${maxMessage}${suffix}`; 155 | pointer += messageSize; 156 | } 157 | 158 | return messages; 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /functions/src/service/dockerhub.ts: -------------------------------------------------------------------------------- 1 | import fetch from 'node-fetch'; 2 | import { settings } from '../config/settings'; 3 | import { Image, ImageType } from '../model/image'; 4 | 5 | export class Dockerhub { 6 | private static get host() { 7 | return settings.dockerhub.host; 8 | } 9 | 10 | public static getRepositoryBaseName() { 11 | return settings.dockerhub.repositoryBaseName; 12 | } 13 | 14 | public static getImageName(imageType: ImageType) { 15 | const { baseImageName, hubImageName, editorImageName } = settings.dockerhub; 16 | switch (imageType) { 17 | case Image.types.base: 18 | return `${baseImageName}`; 19 | case Image.types.hub: 20 | return `${hubImageName}`; 21 | case Image.types.editor: 22 | return `${editorImageName}`; 23 | default: 24 | throw new Error(`[Dockerhub] There is no repository configured of type ${imageType}.`); 25 | } 26 | } 27 | 28 | public static getFullRepositoryName(imageType: ImageType) { 29 | return `${this.getRepositoryBaseName()}/${this.getImageName(imageType)}`; 30 | } 31 | 32 | public static async fetchImageData(imageType: ImageType, tag: string) { 33 | const { host } = this; 34 | const repository = this.getFullRepositoryName(imageType); 35 | const response = await fetch(`${host}/repositories/${repository}/tags/${tag}`); 36 | 37 | if (!response.ok) return null; 38 | 39 | return response.json(); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /functions/src/service/firebase.ts: -------------------------------------------------------------------------------- 1 | import * as admin from 'firebase-admin'; 2 | import { getCredential } from '../config/credential'; 3 | import { setGlobalOptions } from 'firebase-functions/v2/options'; 4 | 5 | const app = admin.initializeApp({ 6 | credential: getCredential(), 7 | databaseURL: 'https://unity-ci-versions.firebaseio.com', 8 | }); 9 | 10 | const auth = app.auth(); 11 | const db = app.firestore(); 12 | 13 | // Set functions to same region as database and hosting 14 | setGlobalOptions({ region: 'europe-west3' }); 15 | 16 | export { admin, auth, db }; 17 | -------------------------------------------------------------------------------- /functions/src/service/github.ts: -------------------------------------------------------------------------------- 1 | import { createAppAuth } from '@octokit/auth-app'; 2 | import { Octokit } from '@octokit/rest'; 3 | import { settings } from '../config/settings'; 4 | import { logger } from 'firebase-functions/v2'; 5 | 6 | export class GitHub { 7 | // https://octokit.github.io/rest.js/v20 8 | static async init(privateKey: string, clientSecret: string): Promise { 9 | const appOctokit = new Octokit({ 10 | authStrategy: createAppAuth, 11 | auth: { 12 | ...settings.github.auth, 13 | privateKey, 14 | clientSecret, 15 | }, 16 | }); 17 | 18 | const { data } = await appOctokit.request('/app'); 19 | logger.debug('app parameters', data); 20 | 21 | return appOctokit; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /functions/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "esModuleInterop": true, 4 | "module": "CommonJS", 5 | "noImplicitReturns": true, 6 | "outDir": "lib", 7 | "sourceMap": true, 8 | "strict": true, 9 | "target": "ES2017" 10 | }, 11 | "compileOnSave": true, 12 | "include": ["src"] 13 | } 14 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "versioning-backend", 3 | "version": "1.0.0", 4 | "description": "Stateful backend to keep track of unity versions and docker build queues", 5 | "main": "src/index.js", 6 | "scripts": { 7 | "lint": "prettier --check \"functions/src/**/*.{js,jsx,ts,tsx}\"", 8 | "format": "prettier --write \"functions/src/**/*.{js,jsx,ts,tsx}\"", 9 | "test": "echo \"No tests yet, feel free to add\" && exit 0" 10 | }, 11 | "repository": { 12 | "type": "git", 13 | "url": "git+https://github.com/game-ci/versioning-backend.git" 14 | }, 15 | "keywords": [ 16 | "unity", 17 | "ci", 18 | "versioning", 19 | "game-ci", 20 | "GameCI" 21 | ], 22 | "author": "GameCI", 23 | "license": "MIT", 24 | "bugs": { 25 | "url": "https://github.com/game-ci/versioning-backend/issues" 26 | }, 27 | "homepage": "https://github.com/game-ci/versioning-backend#readme", 28 | "dependencies": { 29 | "@google-cloud/firestore": "^7.8.0", 30 | "firebase-tools": "^13.10.2", 31 | "firestore-backfire": "^2.5.3", 32 | "prettier": "^2.8.7" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /public/404.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Page Not Found 7 | 8 | 23 | 24 | 25 |
26 |

404

27 |

Page Not Found

28 |

The specified file was not found on this website. Please check the URL for mistakes and try again.

29 |

Why am I seeing this?

30 |

This page was generated by the Firebase Command-Line Interface. To modify it, edit the 404.html file in your project's configured public directory.

31 |
32 | 33 | 34 | -------------------------------------------------------------------------------- /public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Welcome to Firebase Hosting 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 24 | 25 | 26 |

Unity CI Versioning Backend

27 |

Firebase SDK Loading…

28 | 29 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /storage.rules: -------------------------------------------------------------------------------- 1 | rules_version = '2'; 2 | service firebase.storage { 3 | match /b/{bucket}/o { 4 | match /{allPaths=**} { 5 | allow read, write: if request.auth!=null; 6 | } 7 | } 8 | } 9 | --------------------------------------------------------------------------------