├── .github └── workflows │ └── terraform.yml ├── .gitignore ├── .nvmrc ├── .terraform.lock.hcl ├── LICENSE ├── README.md ├── client ├── .gitignore ├── index.html ├── package-lock.json ├── package.json ├── postcss.config.cjs ├── public │ └── favicon.ico ├── src │ ├── App.tsx │ ├── Route.tsx │ ├── SiteTheme.tsx │ ├── api │ │ └── call.ts │ ├── assets │ │ └── react.svg │ ├── auth │ │ ├── authStore.ts │ │ └── useApiToken.ts │ ├── components │ │ ├── base │ │ │ ├── avatar │ │ │ │ └── Avatar.tsx │ │ │ ├── blocks │ │ │ │ ├── CodeBlock.tsx │ │ │ │ └── prism.css │ │ │ ├── buttons │ │ │ │ ├── Button.tsx │ │ │ │ └── IconButton.tsx │ │ │ ├── inputs │ │ │ │ ├── Input.tsx │ │ │ │ └── InputUtils.ts │ │ │ ├── loading │ │ │ │ └── Skeleton.tsx │ │ │ ├── menu │ │ │ │ └── DropdownMenu.tsx │ │ │ ├── nav │ │ │ │ ├── NavTitle.tsx │ │ │ │ ├── Navbar.tsx │ │ │ │ └── Sidebar.tsx │ │ │ └── styles.ts │ │ ├── buttons │ │ │ └── CopyButton.tsx │ │ ├── cards │ │ │ └── DashboardCard.tsx │ │ ├── inputs │ │ │ └── ApiToken.tsx │ │ └── tables │ │ │ └── SortedTable.tsx │ ├── index.css │ ├── lib │ │ ├── StringUtils.ts │ │ └── utils.ts │ ├── main.tsx │ ├── pages │ │ ├── LoginPage.tsx │ │ ├── LogoutPage.tsx │ │ ├── NotFoundPage.tsx │ │ └── dashboard │ │ │ ├── Dashboard.tsx │ │ │ ├── DashboardHomePage.tsx │ │ │ ├── GettingStartedPage.tsx │ │ │ ├── SettingsPage.tsx │ │ │ └── cloudflare_worker.js │ ├── services │ │ └── firebase.ts │ ├── state │ │ ├── createGlobalStore.ts │ │ ├── isLoaded.ts │ │ └── useLocalStorage.ts │ └── vite-env.d.ts ├── tailwind.config.cjs ├── tsconfig.json ├── tsconfig.node.json └── vite.config.ts ├── hcloud-microos-snapshots.pkr.hcl ├── kube.tf ├── kubernetes ├── .env.example ├── README.md ├── create.sh ├── fleet.yaml ├── gcp-secret.yaml ├── get_helm.sh ├── ingress-nginx-annotations.yaml ├── ingress-nginx-configmap.yaml ├── lets-encrypt.yaml ├── services │ ├── README.md │ ├── redis.sh │ └── redis.yaml ├── storage.yaml └── templates │ └── deployment.yaml ├── scripts └── perf_test.mjs ├── server ├── .env.example ├── Dockerfile ├── deploy.sh ├── forward-redis.sh ├── package-lock.json ├── package.json ├── src │ ├── Environment.ts │ ├── ServerAction.ts │ ├── TimeUtils.ts │ ├── api │ │ ├── flush.ts │ │ ├── getMonthlyRenderCounts.ts │ │ ├── getProfile.ts │ │ ├── index.ts │ │ ├── refreshToken.ts │ │ └── render.ts │ ├── browsers │ │ ├── BrowserUtils.ts │ │ └── ChromeBrowser.ts │ ├── db │ │ ├── RedisWrapper.ts │ │ ├── lua │ │ │ ├── deletePattern.lua │ │ │ ├── index.ts │ │ │ └── refreshToken.lua │ │ └── models │ │ │ ├── AbstractHashModel.ts │ │ │ ├── LockModel.ts │ │ │ ├── UrlModel.ts │ │ │ └── UserModel.ts │ ├── doCall.ts │ ├── http │ │ ├── AuthUtils.ts │ │ ├── CallableContext.ts │ │ ├── HttpsError.ts │ │ └── HttpsUtils.ts │ └── index.ts └── tsconfig.json └── shared ├── package-lock.json ├── package.json ├── shared.iml ├── src ├── Action.ts ├── api_schema.json ├── collections │ ├── BiMap.ts │ ├── BiMultimap.ts │ ├── CyclicBuffer.ts │ ├── FilterTrie.ts │ └── Multiset.ts ├── constants.ts ├── index.ts ├── models │ └── User.ts ├── types │ └── ExtraTypes.ts └── utils │ ├── ArrayUtils.ts │ ├── NumberUtils.ts │ └── StringUtils.ts └── tsconfig.json /.github/workflows/terraform.yml: -------------------------------------------------------------------------------- 1 | # This workflow installs the latest version of Terraform CLI and configures the Terraform CLI configuration file 2 | # with an API token for Terraform Cloud (app.terraform.io). On pull request events, this workflow will run 3 | # `terraform init`, `terraform fmt`, and `terraform plan` (speculative plan via Terraform Cloud). On push events 4 | # to the "main" branch, `terraform apply` will be executed. 5 | # 6 | # Documentation for `hashicorp/setup-terraform` is located here: https://github.com/hashicorp/setup-terraform 7 | # 8 | # To use this workflow, you will need to complete the following setup steps. 9 | # 10 | # 1. Create a `main.tf` file in the root of this repository with the `remote` backend and one or more resources defined. 11 | # Example `main.tf`: 12 | # # The configuration for the `remote` backend. 13 | # terraform { 14 | # backend "remote" { 15 | # # The name of your Terraform Cloud organization. 16 | # organization = "example-organization" 17 | # 18 | # # The name of the Terraform Cloud workspace to store Terraform state files in. 19 | # workspaces { 20 | # name = "example-workspace" 21 | # } 22 | # } 23 | # } 24 | # 25 | # # An example resource that does nothing. 26 | # resource "null_resource" "example" { 27 | # triggers = { 28 | # value = "A example resource that does nothing!" 29 | # } 30 | # } 31 | # 32 | # 33 | # 2. Generate a Terraform Cloud user API token and store it as a GitHub secret (e.g. TF_API_TOKEN) on this repository. 34 | # Documentation: 35 | # - https://www.terraform.io/docs/cloud/users-teams-organizations/api-tokens.html 36 | # - https://help.github.com/en/actions/configuring-and-managing-workflows/creating-and-storing-encrypted-secrets 37 | # 38 | # 3. Reference the GitHub secret in step using the `hashicorp/setup-terraform` GitHub Action. 39 | # Example: 40 | # - name: Setup Terraform 41 | # uses: hashicorp/setup-terraform@v1 42 | # with: 43 | # cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }} 44 | 45 | name: 'Terraform' 46 | 47 | on: 48 | push: 49 | branches: [ "main" ] 50 | pull_request: 51 | 52 | permissions: 53 | contents: read 54 | 55 | jobs: 56 | terraform: 57 | name: 'Terraform' 58 | runs-on: ubuntu-latest 59 | environment: production 60 | 61 | # Use the Bash shell regardless whether the GitHub Actions runner is ubuntu-latest, macos-latest, or windows-latest 62 | defaults: 63 | run: 64 | shell: bash 65 | 66 | steps: 67 | # Checkout the repository to the GitHub Actions runner 68 | - name: Checkout 69 | uses: actions/checkout@v3 70 | 71 | # Install the latest version of Terraform CLI and configure the Terraform CLI configuration file with a Terraform Cloud user API token 72 | - name: Setup Terraform 73 | uses: hashicorp/setup-terraform@v1 74 | with: 75 | cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }} 76 | 77 | # Initialize a new or existing Terraform working directory by creating initial files, loading any remote state, downloading modules, etc. 78 | - name: Terraform Init 79 | run: terraform init 80 | 81 | # Checks that all Terraform configuration files adhere to a canonical format 82 | - name: Terraform Format 83 | run: terraform fmt -check 84 | 85 | # Generates an execution plan for Terraform 86 | - name: Terraform Plan 87 | run: terraform plan -input=false 88 | 89 | # On push to "main", build or change infrastructure according to Terraform configuration files 90 | # Note: It is recommended to set up a required "strict" status check in your repository for "Terraform Cloud". See the documentation on "strict" required status checks for more information: https://help.github.com/en/github/administering-a-repository/types-of-required-status-checks 91 | - name: Terraform Apply 92 | if: github.ref == 'refs/heads/"main"' && github.event_name == 'push' 93 | run: terraform apply -auto-approve -input=false 94 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | .terraform 11 | 12 | kubeconfig 13 | k3s-releases.yaml 14 | 15 | node_modules 16 | dist 17 | dist-ssr 18 | *.local 19 | 20 | # Editor directories and files 21 | .vscode/* 22 | !.vscode/extensions.json 23 | .env 24 | client/.env 25 | server/.env 26 | .idea 27 | .DS_Store 28 | *.suo 29 | *.ntvs* 30 | *.njsproj 31 | *.sln 32 | *.sw? 33 | 34 | server/lib 35 | shared/lib 36 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | 18 -------------------------------------------------------------------------------- /.terraform.lock.hcl: -------------------------------------------------------------------------------- 1 | # This file is maintained automatically by "terraform init". 2 | # Manual edits may be lost in future updates. 3 | 4 | provider "registry.terraform.io/hashicorp/cloudinit" { 5 | version = "2.3.2" 6 | hashes = [ 7 | "h1:Vl0aixAYTV/bjathX7VArC5TVNkxBCsi3Vq7R4z1uvc=", 8 | "zh:2487e498736ed90f53de8f66fe2b8c05665b9f8ff1506f751c5ee227c7f457d1", 9 | "zh:3d8627d142942336cf65eea6eb6403692f47e9072ff3fa11c3f774a3b93130b3", 10 | "zh:434b643054aeafb5df28d5529b72acc20c6f5ded24decad73b98657af2b53f4f", 11 | "zh:436aa6c2b07d82aa6a9dd746a3e3a627f72787c27c80552ceda6dc52d01f4b6f", 12 | "zh:458274c5aabe65ef4dbd61d43ce759287788e35a2da004e796373f88edcaa422", 13 | "zh:54bc70fa6fb7da33292ae4d9ceef5398d637c7373e729ed4fce59bd7b8d67372", 14 | "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", 15 | "zh:893ba267e18749c1a956b69be569f0d7bc043a49c3a0eb4d0d09a8e8b2ca3136", 16 | "zh:95493b7517bce116f75cdd4c63b7c82a9d0d48ec2ef2f5eb836d262ef96d0aa7", 17 | "zh:9ae21ab393be52e3e84e5cce0ef20e690d21f6c10ade7d9d9d22b39851bfeddc", 18 | "zh:cc3b01ac2472e6d59358d54d5e4945032efbc8008739a6d4946ca1b621a16040", 19 | "zh:f23bfe9758f06a1ec10ea3a81c9deedf3a7b42963568997d84a5153f35c5839a", 20 | ] 21 | } 22 | 23 | provider "registry.terraform.io/hashicorp/local" { 24 | version = "2.4.0" 25 | constraints = ">= 2.0.0" 26 | hashes = [ 27 | "h1:R97FTYETo88sT2VHfMgkPU3lzCsZLunPftjSI5vfKe8=", 28 | "zh:53604cd29cb92538668fe09565c739358dc53ca56f9f11312b9d7de81e48fab9", 29 | "zh:66a46e9c508716a1c98efbf793092f03d50049fa4a83cd6b2251e9a06aca2acf", 30 | "zh:70a6f6a852dd83768d0778ce9817d81d4b3f073fab8fa570bff92dcb0824f732", 31 | "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", 32 | "zh:82a803f2f484c8b766e2e9c32343e9c89b91997b9f8d2697f9f3837f62926b35", 33 | "zh:9708a4e40d6cc4b8afd1352e5186e6e1502f6ae599867c120967aebe9d90ed04", 34 | "zh:973f65ce0d67c585f4ec250c1e634c9b22d9c4288b484ee2a871d7fa1e317406", 35 | "zh:c8fa0f98f9316e4cfef082aa9b785ba16e36ff754d6aba8b456dab9500e671c6", 36 | "zh:cfa5342a5f5188b20db246c73ac823918c189468e1382cb3c48a9c0c08fc5bf7", 37 | "zh:e0e2b477c7e899c63b06b38cd8684a893d834d6d0b5e9b033cedc06dd7ffe9e2", 38 | "zh:f62d7d05ea1ee566f732505200ab38d94315a4add27947a60afa29860822d3fc", 39 | "zh:fa7ce69dde358e172bd719014ad637634bbdabc49363104f4fca759b4b73f2ce", 40 | ] 41 | } 42 | 43 | provider "registry.terraform.io/hashicorp/null" { 44 | version = "3.2.1" 45 | hashes = [ 46 | "h1:FbGfc+muBsC17Ohy5g806iuI1hQc4SIexpYCrQHQd8w=", 47 | "zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840", 48 | "zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb", 49 | "zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5", 50 | "zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3", 51 | "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", 52 | "zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238", 53 | "zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc", 54 | "zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970", 55 | "zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2", 56 | "zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5", 57 | "zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f", 58 | "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694", 59 | ] 60 | } 61 | 62 | provider "registry.terraform.io/hashicorp/random" { 63 | version = "3.5.1" 64 | hashes = [ 65 | "h1:VSnd9ZIPyfKHOObuQCaKfnjIHRtR7qTw19Rz8tJxm+k=", 66 | "zh:04e3fbd610cb52c1017d282531364b9c53ef72b6bc533acb2a90671957324a64", 67 | "zh:119197103301ebaf7efb91df8f0b6e0dd31e6ff943d231af35ee1831c599188d", 68 | "zh:4d2b219d09abf3b1bb4df93d399ed156cadd61f44ad3baf5cf2954df2fba0831", 69 | "zh:6130bdde527587bbe2dcaa7150363e96dbc5250ea20154176d82bc69df5d4ce3", 70 | "zh:6cc326cd4000f724d3086ee05587e7710f032f94fc9af35e96a386a1c6f2214f", 71 | "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", 72 | "zh:b6d88e1d28cf2dfa24e9fdcc3efc77adcdc1c3c3b5c7ce503a423efbdd6de57b", 73 | "zh:ba74c592622ecbcef9dc2a4d81ed321c4e44cddf7da799faa324da9bf52a22b2", 74 | "zh:c7c5cde98fe4ef1143bd1b3ec5dc04baf0d4cc3ca2c5c7d40d17c0e9b2076865", 75 | "zh:dac4bad52c940cd0dfc27893507c1e92393846b024c5a9db159a93c534a3da03", 76 | "zh:de8febe2a2acd9ac454b844a4106ed295ae9520ef54dc8ed2faf29f12716b602", 77 | "zh:eab0d0495e7e711cca367f7d4df6e322e6c562fc52151ec931176115b83ed014", 78 | ] 79 | } 80 | 81 | provider "registry.terraform.io/hetznercloud/hcloud" { 82 | version = "1.38.2" 83 | constraints = ">= 1.38.2" 84 | hashes = [ 85 | "h1:8q4nC/KLlpevXaSybM5wuJm27UcnEZ/S6kl6Jdw5n+M=", 86 | "zh:12f2fc61812adb02f25be6cc44190904f85e6edbfb38f8dec9c222b619bfc03a", 87 | "zh:1c4d50c001245467a453d9986f89e9df678830a943a9b7a5d7062655d60924de", 88 | "zh:2d85bd7b4ba8e115a0a4e0023bb029fe6b7ec19241ad5551d76ed8fb988c4183", 89 | "zh:31a1884b4225236eb30723d642d4c7a2d2effbeef512bba1005f28d1aaff1cd3", 90 | "zh:3a5c42926e2b495bca9a8109449ecb9313f43072159fbf9372b9e067216f4609", 91 | "zh:4c4e8f2c7f5510d56a826ed267e3d0723b5123737e415302e052296fa80a7601", 92 | "zh:5769c6422990ea3acb4bef12133ce12338a60a9323653b48aa9a29a1c7a03d36", 93 | "zh:590893321c68df5081a5853fcccfe4bef5caf32d42697fbd370c7477f06fb468", 94 | "zh:63b67b0358ac4e3405e8871771196fd359f819f0ba662608746ddb4de0850b46", 95 | "zh:69674d96742f181b91afea576806ecf86a5eb38b9ad27d78b9023dc0983b8c26", 96 | "zh:7a9ea0a171596bdca01773b53742d514620477fd486acdd8d1c9010a57e4ffe7", 97 | "zh:bfaa433238fd3cefcc8135dc927159cb03bbe4e1e53938b81c27bbe707329b9a", 98 | "zh:e57aa989fccb60cef9eae2c820e2c2611f5b4563913853633d5756b040ca1cbf", 99 | "zh:fdaa4653b588a525cc651a4077c36020cb415201d797e072e4e6c88aa52fad6a", 100 | ] 101 | } 102 | 103 | provider "registry.terraform.io/integrations/github" { 104 | version = "5.23.0" 105 | constraints = ">= 4.0.0" 106 | hashes = [ 107 | "h1:FSHiqYsBDKUZi5I9q4ozZ59/sMTTQl1GYXeILLZO6oA=", 108 | "zh:006d889fedb143523e2b634478422643d8979b554ff54e8e14985e2ba1e64a55", 109 | "zh:13af3551f5f13ab5558d4d90e8ca4a43fccd0d09aa7dbf04ed8a599ac9aadbf1", 110 | "zh:142144634d525eff5670ac292fd3ed019f504b51974233ecad852cafcc73e2eb", 111 | "zh:168ad7577bc9f181361e5015642dbb030f33387b4a6284b9855861d81f1ec341", 112 | "zh:45aa1df2ccdf2092597507b72197b8e35b2fa651b2338d8b68248f93267f01d6", 113 | "zh:48209af2abf45ac2a8d45b804e0e6c542c48d8eaac44a31f5f70db402e2dc4e6", 114 | "zh:4da802b73275ea823c6f8325ab2b7b65650d212884d41f0295135e337213fb20", 115 | "zh:6d862eedf87808955eb57376dc4a4029f74ec9281f9612406fff7b9e09796985", 116 | "zh:80efa1ea1d196c1ca35899e88c8935f41af10d955bfce9745994762654549802", 117 | "zh:947016c71515fb7725c2ac201fcd03a8dd08dc2c6e8aa34af398b8d43272d21d", 118 | "zh:a4ae425291ed246e17cff587f7364c813f191ac43917c6a25e32a2137e92f664", 119 | "zh:c962de2984083905d4c6afb3697e800d281b3ba2b042da806b809d3afb8f4a23", 120 | "zh:cd6e49f8967fc59fee76606bf7ad5efe431cc02ee797dc490b644fb774dcb363", 121 | "zh:fc50b9a38f8f6cb4dcb33aa1cef63bc7b706df2f6ae84e1d7db7481170693947", 122 | ] 123 | } 124 | 125 | provider "registry.terraform.io/tenstad/remote" { 126 | version = "0.1.1" 127 | constraints = ">= 0.0.23" 128 | hashes = [ 129 | "h1:x+VrpVS5+DG1ARCzq8m6yVylQYox1WUqGLR1VAgf+CE=", 130 | "zh:00dab99d546d4f74b9b9a4aa23e5e4562edee95f815d284c1ffda92bab6e8b9f", 131 | "zh:12475396d20d34ccdb5132caf0a50378ffc46f28fa8c6e5b867ac505ea6a776c", 132 | "zh:3c6c0c664d5f7c4d3feb28d841b627781381f9e439a2f834fc64b5ddd577ce8f", 133 | "zh:65f0f405b0ec692c40d718a27dd21ac991a9e8dd5a9c8fc08df72ffd6f540c7f", 134 | "zh:6f214da8658ff479714259c3a29d089d79889a70162e27bbdd1d7636cf9b2361", 135 | "zh:701753bdfe5a64aa354f4c418ccad3f77c09030b737685895e95907cddd87ffb", 136 | "zh:7095c82c5008d56d9b315cf43b07c7f2c86de2f608315d2941e2a0649fb968d3", 137 | "zh:70d370bd9d7224df5ef73fc2152e74df25796ba69701406c740b1c584d4eb7b6", 138 | "zh:a95426eb286c34c9a173c4ff03e45eecb56ec095783b4acb3e0c9cadbf7aaa42", 139 | "zh:b7032268227a640805a4f3c7440180db0362a2497a24ddfb1c8293d403e1c24c", 140 | "zh:bbf660b0d8a72dcce3510e9c6708f01f045e4259046984c2c070727ae44d2cf6", 141 | "zh:c825fdb5cf3328260e9db8aa711aba515d04277086f9e4556314b1a1f7265cc4", 142 | "zh:cf9d8fd66f71fe7f5eef983eca06f209ef366bb32dcc956b2231dc8eff681dcb", 143 | "zh:ef75990a7a5282f12e1b7b4b4e34419103bb2d05a3ece39d921f42a29dcad476", 144 | "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", 145 | ] 146 | } 147 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Acorn1010 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Prerender service based on https://prerender.io 2 | 3 | **NOTE:** All commits should use https://www.conventionalcommits.org/en/v1.0.0/. 4 | **TODO(acorn1010):** Use https://github.com/semantic-release/semantic-release 5 | 6 | # Prerequisites 7 | You'll need the following dependencies before you can run this project: 8 | * CLI tools: 9 | * Ubuntu:
10 | * `sudo apt install -y packer kubectl hcloud-cli` 11 | * Go to https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli#install-terraform and install Terraform 12 | * Mac: `brew install terraform packer kubectl hcloud` 13 | * Make an `ed25519` SSH key (`rsa` isn't supported):
14 | `[ -f ~/.ssh/id_ed25519 ] || ssh-keygen -t ed25519 -N "" -f ~/.ssh/id_ed25519 && chmod 400 ~/.ssh/id_ed25519*` 15 | * Set up an hcloud context: `hcloud context create ` 16 | * Add this Bash alias:
17 | `alias createkh='tmp_script=$(mktemp) && curl -sSL -o "${tmp_script}" https://raw.githubusercontent.com/kube-hetzner/terraform-hcloud-kube-hetzner/master/scripts/create.sh && chmod +x "${tmp_script}" && "${tmp_script}" && rm "${tmp_script}"'` 18 | 19 | # Deployment 20 | * Run `createkh` from your repository root. Press `Enter` to skip directory choice, then `yes` when creating the MicroOS snapshots. 21 | 22 | # Tech Stack 23 | * Hetzner for K3s cluster (CPX31) 24 | * Terraform 25 | * **client:** 26 | * [Vite](https://vitejs.dev/) + TypeScript SPA + [Tailwind](https://tailwindcss.com/) + [Redis](https://redis.io/) + [Shadcn UI](ui.shadcn.com/) 27 | * **server:** 28 | * **Compute:** Hetzner shared cloud (cheap compute + network) 29 | * Alternatives: 30 | * Contabo (cheap compute) 31 | * OVH (cheap network) 32 | * **Storage:** Hetzner volumes 33 | * Alternatives: 34 | * [Cloudflare R2](https://www.cloudflare.com/products/r2/) ($15 / TB, $4.50 / 1M writes, $0.36 / 1M reads) 35 | * [Wasabi](https://wasabi.com/cloud-storage-pricing) ($6 / TB, no egress) 36 | * **Network:** Hetzner Load Balancer 37 | * **CDN:** Cloudflare Pages (it's free) 38 | * Playwright for browser rendering 39 | * Worker scripts for clients to interface with API: 40 | * [https://github.com/prerender/prerender-cloudflare-worker/blob/master/index.js#L154](https://github.com/prerender/prerender-cloudflare-worker/blob/master/index.js#L154) 41 | * **shared** 42 | * A repo shared by both client and server. Safe way to allow typed APIs without leaking server code. `ts-json-schema-generator` is used for validation on the server. 43 | 44 | # Deployment 45 | 1. Create a Terraform user API token [here](https://app.terraform.io) and set up an `organization` and `workspace`. This is your `TF_API_TOKEN`. 46 | 2. Get your `SSH_PUBLIC_KEY` from your https://github.com/acorn1010.keys 47 | 2. Add `SSH_PUBLIC_KEY`, `TF_API_TOKEN`, `GCR_SECRET`, `HETZNER_TOKEN` to your `GitHub Settings > Secrets and Variables > New repository secret` 48 | 49 | # Overview 50 | * Backend Service on K8 handles API requests to render pages, store the pages in our Storage solution 51 | -------------------------------------------------------------------------------- /client/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | 15 | # Editor directories and files 16 | .vscode/* 17 | !.vscode/extensions.json 18 | .idea 19 | .DS_Store 20 | *.suo 21 | *.ntvs* 22 | *.njsproj 23 | *.sln 24 | *.sw? 25 | -------------------------------------------------------------------------------- /client/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | RenderMy.Site - Dynamic Rendering for JavaScript Websites 10 | 11 | 12 | 13 |
14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /client/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "client", 3 | "private": true, 4 | "version": "0.0.1", 5 | "type": "module", 6 | "scripts": { 7 | "dev": "vite", 8 | "build": "tsc && vite build", 9 | "deploy": "npm i && npm run build", 10 | "preview": "vite preview" 11 | }, 12 | "dependencies": { 13 | "@fontsource/roboto": "^4.5.8", 14 | "@loadable/component": "^5.15.3", 15 | "@radix-ui/react-avatar": "^1.0.2", 16 | "@radix-ui/react-dropdown-menu": "^2.0.4", 17 | "@types/prismjs": "^1.26.0", 18 | "class-variance-authority": "^0.5.2", 19 | "clsx": "^1.2.1", 20 | "firebase": "^9.18.0", 21 | "lucide-react": "^0.127.0", 22 | "minimatch": "^7.4.3", 23 | "moderndash": "^3.1.0", 24 | "prismjs": "^1.29.0", 25 | "react": "^18.2.0", 26 | "react-dom": "^18.2.0", 27 | "react-helmet": "^6.1.0", 28 | "react-icons": "^4.8.0", 29 | "tailwind-merge": "^1.10.0", 30 | "tailwindcss-animate": "^1.0.5", 31 | "wouter": "^2.10.0", 32 | "zustand": "^4.3.6" 33 | }, 34 | "devDependencies": { 35 | "@types/loadable__component": "^5.13.4", 36 | "@types/react": "^18.0.29", 37 | "@types/react-dom": "^18.0.11", 38 | "@vitejs/plugin-react": "^3.1.0", 39 | "autoprefixer": "^10.4.14", 40 | "postcss": "^8.4.21", 41 | "tailwindcss": "^3.3.1", 42 | "typescript": "^5.0.4", 43 | "vite": "^4.2.1" 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /client/postcss.config.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | } 7 | -------------------------------------------------------------------------------- /client/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorn1010/render/9f1001e3c8061dca255d74cfaa575ccfc6c7424b/client/public/favicon.ico -------------------------------------------------------------------------------- /client/src/App.tsx: -------------------------------------------------------------------------------- 1 | import {useAuth} from "./auth/authStore"; 2 | import {SiteTheme} from "./SiteTheme"; 3 | import {Switch} from "wouter"; 4 | import Dashboard from "@/pages/dashboard/Dashboard"; 5 | import * as React from "react"; 6 | import {AuthRoute, Route} from "@/Route"; 7 | 8 | export function App() { 9 | return ( 10 | <> 11 | 12 | 13 | 14 | import('./pages/LoginPage')} /> 15 | import('./pages/LogoutPage')} /> 16 | import('./pages/NotFoundPage')} /> 17 | 18 | {/* Dashboard routes. Because dashboard is at the root, we need a catch-all */} 19 | 20 | 21 | 22 | 23 | ); 24 | } 25 | 26 | function AuthUpdater() { 27 | useAuth(); 28 | return null; 29 | } 30 | -------------------------------------------------------------------------------- /client/src/Route.tsx: -------------------------------------------------------------------------------- 1 | import {PropsWithChildren, useMemo} from "react"; 2 | import loadable from "@loadable/component"; 3 | import {Redirect, Route as Woute} from "wouter"; 4 | import * as React from "react"; 5 | import {authStore} from "@/auth/authStore"; 6 | import {isLoaded} from "@/state/isLoaded"; 7 | 8 | type RouteProps = PropsWithChildren<{path?: `/${string}`, lazy?: () => Promise<{default: () => JSX.Element}>}>; 9 | 10 | export function Route(props: RouteProps) { 11 | const {children, lazy, ...rest} = props; 12 | const LoadableComponent = useMemo(() => lazy ? loadable(lazy) : null, [lazy]); 13 | 14 | return {LoadableComponent ? : children}; 15 | } 16 | 17 | export function AuthRoute({type, ...rest}: RouteProps & {type: 'guest' | 'auth'}) { 18 | const userId = authStore.use('userId')[0]; 19 | 20 | if (!isLoaded(userId)) { 21 | return

Loading...

; 22 | } else if (type === 'guest' && !!userId) { 23 | // Authenticated. Show the homepage. 24 | return ; 25 | } else if (type === 'auth' && !userId) { 26 | // Unauthenticated. 27 | return ; 28 | } 29 | 30 | return ; 31 | } 32 | -------------------------------------------------------------------------------- /client/src/SiteTheme.tsx: -------------------------------------------------------------------------------- 1 | import React, {PropsWithChildren} from "react"; 2 | import './index.css' 3 | import '@fontsource/roboto/500.css'; 4 | import '@fontsource/roboto/700.css'; 5 | 6 | export function SiteTheme({children}: PropsWithChildren<{}>) { 7 | return <>{children}; 8 | } 9 | -------------------------------------------------------------------------------- /client/src/api/call.ts: -------------------------------------------------------------------------------- 1 | import {Actions} from "@shared/Action"; 2 | import {useEffect} from "react"; 3 | import {authStore} from "@/auth/authStore"; 4 | import {createGlobalStore} from "@/state/createGlobalStore"; 5 | import {Multiset} from "@shared/collections/Multiset"; 6 | 7 | type CallArgs = Actions[T]['input'] extends any[] 8 | ? Actions[T]['input'] 9 | : {} extends Actions[T]['input'] ? [] : [Actions[T]['input']]; 10 | 11 | /** 12 | * Interval in milliseconds of long-polling. Determines how frequently data is polled from the 13 | * server, where lower values mean more frequent polling. 14 | */ 15 | const LONG_POLL_INTERVAL_MS = 10_000; 16 | 17 | const pollStore = createGlobalStore({} as {[key: string]: any}); 18 | 19 | class PollSingleton { 20 | /** Maps a JSON.stringify() of action + props -> number of listeners. */ 21 | private readonly refs = new Multiset(); 22 | 23 | /** 24 | * Maps a key (JSON.stringify()) of action + props -> interval timer. Use this when removing a 25 | * listener. 26 | */ 27 | private readonly unsubscribePaths = new Map>(); 28 | 29 | connect(action: T, ...args: CallArgs) { 30 | const key = JSON.stringify([action, ...args]); 31 | if (this.refs.add(key) !== 1) { 32 | return; // Not the first connection. No need to establish a listener. 33 | } 34 | 35 | // This is the first connection. Set up a long-polling listener. 36 | const refresh = async () => { 37 | try { 38 | // NOTE: It's possible that we've updated the client's state in the time we've started 39 | // fetching from the server. If so, don't update the local state. 40 | const oldState = pollStore.get(key); 41 | const result = await call[action](...args); 42 | if (oldState === pollStore.get(key)) { 43 | pollStore.set(key, result); // State is still the same. It's safe to replace! 44 | } 45 | } catch (e) { 46 | console.error(`Failed to do long poll for ${action} with args: ${args}`, e); 47 | } 48 | }; 49 | this.unsubscribePaths.set(key, setInterval(refresh, LONG_POLL_INTERVAL_MS)); 50 | refresh().then(() => {}); 51 | } 52 | 53 | close(action: T, ...args: CallArgs) { 54 | const key = JSON.stringify([action, ...args]); 55 | if (!this.refs.remove(key)) { 56 | throw new Error(`Tried to close a connection that was never opened. Did you mutate poll.use args?: ${key}`); 57 | } 58 | if (!this.refs.has(key)) { 59 | clearInterval(this.unsubscribePaths.get(key)!); 60 | } 61 | } 62 | } 63 | /** Tracks the open connections, deduping poll requests where necessary. */ 64 | const pollSingleton = new PollSingleton(); 65 | 66 | /** 67 | * Allows periodic polling of state from the server. This is essentially a "slow" pub/sub without 68 | * SSE or websockets. 69 | */ 70 | export const poll = { 71 | /** 72 | * Updates the long-poll data for `action` with the given `args`. This is a temporary update. 73 | * After a timeout greater than `LONG_POLL_INTERVAL_MS` (or during the next long-poll), this 74 | * update will be cleared and replaced with the long-poll data. 75 | */ 76 | update: (action: T, ...args: CallArgs) => (data: Partial) => { 77 | const key = JSON.stringify([action, ...args]); 78 | const scope: {ref: Partial} = {ref: {}}; 79 | pollStore.set(key, prevState => { 80 | scope.ref = ({...prevState, ...data}); 81 | return scope.ref; 82 | }); 83 | setTimeout(() => { 84 | if (scope.ref === pollStore.get(key)) { 85 | // State is stale and hasn't changed. Delete it! 86 | pollStore.delete(key); 87 | } 88 | }, LONG_POLL_INTERVAL_MS * 1.5); 89 | }, 90 | 91 | /** 92 | * Periodically polls `action` with the given `args`. Returns `undefined` until the API has had a 93 | * chance to reply. 94 | * 95 | * TODO(acorn1010): Add Multiset so we can support batching in case multiple components need access 96 | * to the same endpoint. 97 | */ 98 | use: (action: T, ...args: CallArgs): Actions[T]['output'] | undefined => { 99 | const key = JSON.stringify([action, ...args]); 100 | const [result] = pollStore.use(key); 101 | 102 | useEffect(() => { 103 | pollSingleton.connect(action, ...args); 104 | return () => { 105 | setTimeout(() => pollSingleton.close(action, ...args), LONG_POLL_INTERVAL_MS); 106 | }; 107 | }, [action, ...args]); 108 | 109 | return result; 110 | }, 111 | }; 112 | 113 | /** 114 | * Calls the server API. The call will be directed to the server for a room if the player is in a 115 | * room. Otherwise, the call will be load-balanced and will hit any server. 116 | */ 117 | export const call: {[K in keyof Actions]: (...args: CallArgs) => Promise} = 118 | new Proxy({}, { 119 | get: (_: unknown, key: K) => { 120 | return (...args: CallArgs) => _call(key, ...args); 121 | }, 122 | }) as any; 123 | 124 | async function _call( 125 | action: T, ...args: CallArgs): Promise { 126 | const url = import.meta.env.DEV ? 'http://localhost:3000/api' : 'https://api.rendermy.site/api'; 127 | authToken = await queryAuthToken(); 128 | const response = await fetch(url, { 129 | method: 'POST', 130 | body: JSON.stringify({a: action, d: args ?? null}), 131 | headers: authToken ? { 132 | Authorization: `Bearer ${authToken}`, 133 | 'Content-Type': 'application/json', 134 | } : {}, 135 | }); 136 | // API returns JSON with 'd' for data, and 'e' for error on success 137 | const result = await response.json() as {d: any, e: string}; 138 | if (result.e) { // This was an error 139 | throw new Error(result.e); 140 | } 141 | return result.d as Actions[T]['output']; 142 | } 143 | 144 | /** The last JWT auth bearer token that was found. Updated when calling room. */ 145 | let authToken = ''; 146 | 147 | /** Waits for an auth token to be available, then sets the authToken. */ 148 | function queryAuthToken(): Promise { 149 | return new Promise(async (resolve) => { 150 | do { 151 | const user = authStore.get('user'); 152 | // Always requery the authToken. This ensures our token doesn't go stale. 153 | authToken = await user?.getIdToken() || authToken; 154 | if (!authToken) { 155 | // Wait a bit between requests. 156 | await new Promise(resolve2 => setTimeout(resolve2, 50)); 157 | } 158 | } while (!authToken); 159 | resolve(authToken); 160 | }); 161 | } 162 | -------------------------------------------------------------------------------- /client/src/assets/react.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /client/src/auth/authStore.ts: -------------------------------------------------------------------------------- 1 | import {createGlobalStore} from "@/state/createGlobalStore"; 2 | import {type Auth, type User, type AuthProvider, signOut, getAuth, GithubAuthProvider, GoogleAuthProvider, signInWithPopup} from "firebase/auth"; 3 | import {useEffect} from "react"; 4 | import {app} from "@/services/firebase"; 5 | import {getLocalStorage, setLocalStorage, useLocalStorage} from "@/state/useLocalStorage"; 6 | import {isEqual} from "moderndash"; 7 | 8 | const initialState = { 9 | user: undefined as Pick | null | undefined, 10 | 11 | /** 12 | * userId starts as undefined. If there's no logged-in user, this will be null. It's only 13 | * undefined while loading. 14 | */ 15 | userId: undefined as string | undefined | null, 16 | email: undefined as string | undefined | null, 17 | displayName: undefined as string | undefined | null, 18 | 19 | /** URL to this user's photo, if any (e.g. "https://foony.com/foony.png") */ 20 | photoUrl: undefined as string | undefined | null, 21 | } as const; 22 | 23 | /** In order to use authStore, you must include #useAuth higher in the DOM. */ 24 | export const authStore = createGlobalStore(initialState); 25 | 26 | let auth: Auth | null = null; 27 | /** Initializes auth if it hasn't already been initialized. Must be included if using authStore. */ 28 | export function useAuth() { 29 | // Load the old auth from local storage, if any. Lowers time-to-interactive 30 | const [oldAuth] = useLocalStorage | null>('auth', null); 31 | 32 | // On initial load, set the old auth to the store if it exists. 33 | useEffect(() => { 34 | if (!auth && oldAuth) { 35 | auth = getAuth(app); 36 | updateStore({ 37 | email: oldAuth.email ?? null, 38 | displayName: oldAuth.displayName ?? null, 39 | uid: oldAuth.userId ?? '', 40 | photoURL: oldAuth.photoUrl ?? null, 41 | getIdToken(forceRefresh?: boolean): Promise { 42 | let unsubscribe: () => void = () => {}; 43 | return new Promise((resolve, reject) => { 44 | if (auth) { 45 | unsubscribe = auth.onAuthStateChanged(user => resolve(user?.getIdToken(forceRefresh) ?? '')); 46 | } else { 47 | reject(new Error('Auth not initialized.')); 48 | } 49 | }).finally(unsubscribe); 50 | }, 51 | }); 52 | } 53 | }, [oldAuth]); 54 | 55 | useEffect(() => { 56 | if (!auth) { 57 | auth = getAuth(app); 58 | } 59 | // NOTE: There seems to be a bug with Firestore where onAuthStateChange removes the user from 60 | // the store until the callback is called. This means that if you're logged in, and you refresh 61 | // while loading, you'll be logged out. 62 | // Only happens on Firefox? 63 | auth.onAuthStateChanged((user) => { 64 | updateStore(user); 65 | }); 66 | }, []); 67 | } 68 | 69 | /** Logs out the currently-logged-in user. */ 70 | export function logout(): Promise { 71 | if (!auth) { 72 | auth = getAuth(app); 73 | } 74 | return signOut(auth); 75 | } 76 | 77 | const PROVIDERS = { 78 | google: () => new GoogleAuthProvider(), 79 | github: () => new GithubAuthProvider(), 80 | } as const satisfies {[provider: string]: () => AuthProvider}; 81 | export type AuthProviderId = keyof typeof PROVIDERS; 82 | 83 | /** Opens a popup that allows the user to sign in with the given provider. */ 84 | export async function signInWithProvider(providerKey: AuthProviderId): Promise { 85 | try { 86 | const result = await signInWithPopup(getAuth(), PROVIDERS[providerKey]()); 87 | updateStore(result.user); 88 | } catch (e) { 89 | console.error('Error signing in!', e); 90 | } 91 | } 92 | 93 | function updateStore(user: Pick | null) { 94 | const sharedState = { 95 | userId: user?.uid ?? null, 96 | email: user?.email ?? null, 97 | displayName: user?.displayName ?? null, 98 | photoUrl: user?.photoURL ?? null, 99 | } as const; 100 | if (!isEqual(getLocalStorage('auth'), sharedState)) { 101 | setLocalStorage>('auth', sharedState); 102 | } 103 | authStore.update({user, ...sharedState}); 104 | } 105 | -------------------------------------------------------------------------------- /client/src/auth/useApiToken.ts: -------------------------------------------------------------------------------- 1 | import {call, poll} from "@/api/call"; 2 | import {useEffect} from "react"; 3 | import {isLoaded} from "@/state/isLoaded"; 4 | 5 | /** 6 | * Returns the user's API token. If the user doesn't have an API token, calls the server and 7 | * requests one. 8 | */ 9 | export function useApiToken() { 10 | const profile = poll.use('getProfile'); 11 | // TODO(acorn1010): Maybe request a new token when the user is created on the server. 12 | 13 | const isProfileLoaded = isLoaded(profile); 14 | const token = profile?.token; 15 | useEffect(() => { 16 | if (!isProfileLoaded || token) { 17 | return; 18 | } 19 | (async () => { 20 | // If profile is loaded and the user doesn't have a token, refresh their token 21 | const token = await call.refreshToken(); 22 | if (token) { 23 | poll.update('getProfile')({token}); 24 | } 25 | })(); 26 | }, [isProfileLoaded, token]); 27 | 28 | return profile?.token ?? ''; 29 | } 30 | -------------------------------------------------------------------------------- /client/src/components/base/avatar/Avatar.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react" 2 | import * as AvatarPrimitive from "@radix-ui/react-avatar" 3 | 4 | import { cn } from "@/lib/utils" 5 | import {PropsWithChildren} from "react"; 6 | 7 | type InnerAvatarProps = 8 | Pick, 'className' | 'onClick'> & { 9 | src?: string, 10 | }; 11 | 12 | type AvatarProps = PropsWithChildren; 13 | 14 | const avatarStyle = cn` 15 | relative 16 | flex 17 | h-10 18 | w-10 19 | shrink-0 20 | overflow-hidden 21 | rounded-full 22 | 23 | focus:outline-none 24 | focus:ring-2 25 | focus:ring-slate-400 26 | focus:ring-offset-2 27 | `; 28 | 29 | export const Avatar = React.forwardRef, AvatarProps>((props, ref) => { 30 | const {children, className, src, ...rest} = props; 31 | return ( 32 | 33 | 34 | {children && {children}} 35 | 36 | ); 37 | }); 38 | Avatar.displayName = AvatarPrimitive.Root.displayName 39 | 40 | const AvatarImage = React.forwardRef< 41 | React.ElementRef, 42 | React.ComponentPropsWithoutRef 43 | >(({ className, ...props }, ref) => ( 44 | 49 | )) 50 | AvatarImage.displayName = AvatarPrimitive.Image.displayName 51 | 52 | const AvatarFallback = React.forwardRef< 53 | React.ElementRef, 54 | React.ComponentPropsWithoutRef 55 | >(({ className, ...props }, ref) => ( 56 | 64 | )) 65 | AvatarFallback.displayName = AvatarPrimitive.Fallback.displayName 66 | -------------------------------------------------------------------------------- /client/src/components/base/blocks/CodeBlock.tsx: -------------------------------------------------------------------------------- 1 | import {PropsWithChildren, useEffect, useRef} from "react"; 2 | import Prism from "prismjs"; 3 | import {CopyButton} from "@/components/buttons/CopyButton"; 4 | import {cn} from "@/lib/utils"; 5 | import './prism.css'; 6 | 7 | type CodeBlockProps = PropsWithChildren<{classes?: {root?: string, pre?: string, code?: string}, code: string, language: 'javascript'}>; 8 | export function CodeBlock({classes, code, language}: CodeBlockProps) { 9 | const codeRef = useRef(null); 10 | 11 | const element = codeRef.current; 12 | useEffect(() => { 13 | if (element) { 14 | Prism.highlightElement(element); 15 | } 16 | }, [code, element]); 17 | 18 | return ( 19 |
20 |
21 |

{language}

22 | 23 |
24 | 
25 |           {code}
26 |
27 |
28 | ); 29 | } 30 | -------------------------------------------------------------------------------- /client/src/components/base/buttons/Button.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react" 2 | import { VariantProps, cva } from "class-variance-authority" 3 | import { cn } from "@/lib/utils" 4 | import {focusStyle} from "@/components/base/styles"; 5 | 6 | const buttonVariants = cva( 7 | cn( 8 | "active:scale-95 inline-flex items-center justify-center rounded-md text-sm font-medium transition-colors dark:hover:bg-slate-800 disabled:opacity-50 disabled:pointer-events-none data-[state=open]:bg-slate-100 dark:data-[state=open]:bg-slate-800", 9 | focusStyle, 10 | ), 11 | { 12 | variants: { 13 | variant: { 14 | default: 15 | "bg-slate-900 text-white hover:bg-slate-700 dark:bg-blue-300 dark:hover:bg-blue-200 dark:active:bg-blue-400 dark:text-slate-900", 16 | destructive: 17 | "bg-red-500 text-white hover:bg-red-600 dark:hover:bg-red-600", 18 | outline: 19 | "bg-transparent border border-slate-200 hover:bg-slate-100 dark:border-slate-700 dark:text-slate-100", 20 | subtle: 21 | "bg-slate-100 text-slate-900 hover:bg-slate-200 dark:bg-slate-700 dark:text-slate-100", 22 | ghost: 23 | "bg-transparent hover:bg-slate-100 dark:hover:bg-slate-800 dark:text-slate-100 dark:hover:text-slate-100 data-[state=open]:bg-transparent dark:data-[state=open]:bg-transparent", 24 | link: "bg-transparent dark:bg-transparent underline-offset-4 hover:underline text-slate-900 dark:text-slate-100 hover:bg-transparent dark:hover:bg-transparent", 25 | }, 26 | size: { 27 | default: "h-10 py-2 px-4", 28 | sm: "h-9 px-2 rounded-md", 29 | lg: "h-11 px-8 rounded-md", 30 | }, 31 | }, 32 | defaultVariants: { 33 | variant: "default", 34 | size: "default", 35 | }, 36 | } 37 | ) 38 | 39 | export type ButtonProps = 40 | Pick< 41 | React.ButtonHTMLAttributes & VariantProps, 42 | 'children' | 'className' | 'disabled' | 'variant' | 'size' | 'onClick'>; 43 | 44 | const Button = React.forwardRef( 45 | ({ className, variant, size, ...props }, ref) => { 46 | return ( 47 | 12 | ); 13 | } 14 | 15 | const iconStyle = cn` 16 | rounded 17 | h-8 18 | w-8 19 | p-2 20 | text-zinc-500 21 | 22 | cursor-pointer 23 | hover:bg-zinc-800 24 | hover:text-white 25 | active:bg-zinc-900 26 | `; 27 | -------------------------------------------------------------------------------- /client/src/components/base/inputs/Input.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react" 2 | 3 | import { cn } from "@/lib/utils" 4 | 5 | export type InputProps = 6 | Pick, 'className' | 'type' | 'value' | 'readOnly'> 7 | 8 | const Input = React.forwardRef( 9 | ({ className, ...props }, ref) => { 10 | return ( 11 | 19 | ) 20 | } 21 | ) 22 | Input.displayName = "Input" 23 | 24 | export { Input } 25 | -------------------------------------------------------------------------------- /client/src/components/base/inputs/InputUtils.ts: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | /** 4 | * Wraps an input's onKeyDown event and calls "callback" if enter was pressed. If the event was a keyboard event and was 5 | * "Enter", then the event is also prevented. 6 | * @param callback a callback that will be triggered if enter is pressed. 7 | */ 8 | export const onEnter = (callback: (e: React.KeyboardEvent) => void | Promise) => (e: React.SyntheticEvent) => { 9 | if (!('keyCode' in e)) { 10 | return; 11 | } 12 | const keyboardEvent = e as React.KeyboardEvent; 13 | if (keyboardEvent.key === 'Enter') { 14 | keyboardEvent.preventDefault(); 15 | return callback(keyboardEvent); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /client/src/components/base/loading/Skeleton.tsx: -------------------------------------------------------------------------------- 1 | import {cn} from "@/lib/utils"; 2 | 3 | /** Provides a simple rectangular skeleton for use when a component is loading. */ 4 | export function Skeleton({className}: { className?: string }) { 5 | return
; 6 | } 7 | -------------------------------------------------------------------------------- /client/src/components/base/menu/DropdownMenu.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react" 2 | import * as DropdownMenuPrimitive from "@radix-ui/react-dropdown-menu" 3 | import { Check, ChevronRight, Circle } from "lucide-react" 4 | 5 | import { cn } from "@/lib/utils" 6 | 7 | const DropdownMenu = DropdownMenuPrimitive.Root 8 | 9 | const DropdownMenuTrigger = DropdownMenuPrimitive.Trigger 10 | 11 | const DropdownMenuGroup = DropdownMenuPrimitive.Group 12 | 13 | const DropdownMenuPortal = DropdownMenuPrimitive.Portal 14 | 15 | const DropdownMenuSub = DropdownMenuPrimitive.Sub 16 | 17 | const DropdownMenuRadioGroup = DropdownMenuPrimitive.RadioGroup 18 | 19 | const DropdownMenuSubTrigger = React.forwardRef< 20 | React.ElementRef, 21 | React.ComponentPropsWithoutRef & { 22 | inset?: boolean 23 | } 24 | >(({ className, inset, children, ...props }, ref) => ( 25 | 34 | {children} 35 | 36 | 37 | )) 38 | DropdownMenuSubTrigger.displayName = 39 | DropdownMenuPrimitive.SubTrigger.displayName 40 | 41 | const DropdownMenuSubContent = React.forwardRef< 42 | React.ElementRef, 43 | React.ComponentPropsWithoutRef 44 | >(({ className, ...props }, ref) => ( 45 | 53 | )) 54 | DropdownMenuSubContent.displayName = 55 | DropdownMenuPrimitive.SubContent.displayName 56 | 57 | const DropdownMenuContent = React.forwardRef< 58 | React.ElementRef, 59 | React.ComponentPropsWithoutRef 60 | >(({ className, sideOffset = 4, ...props }, ref) => ( 61 | 62 | 71 | 72 | )) 73 | DropdownMenuContent.displayName = DropdownMenuPrimitive.Content.displayName 74 | 75 | const DropdownMenuItem = React.forwardRef< 76 | React.ElementRef, 77 | React.ComponentPropsWithoutRef & { 78 | inset?: boolean 79 | } 80 | >(({ className, inset, ...props }, ref) => ( 81 | 90 | )) 91 | DropdownMenuItem.displayName = DropdownMenuPrimitive.Item.displayName 92 | 93 | const DropdownMenuCheckboxItem = React.forwardRef< 94 | React.ElementRef, 95 | React.ComponentPropsWithoutRef 96 | >(({ className, children, checked, ...props }, ref) => ( 97 | 106 | 107 | 108 | 109 | 110 | 111 | {children} 112 | 113 | )) 114 | DropdownMenuCheckboxItem.displayName = 115 | DropdownMenuPrimitive.CheckboxItem.displayName 116 | 117 | const DropdownMenuRadioItem = React.forwardRef< 118 | React.ElementRef, 119 | React.ComponentPropsWithoutRef 120 | >(({ className, children, ...props }, ref) => ( 121 | 129 | 130 | 131 | 132 | 133 | 134 | {children} 135 | 136 | )) 137 | DropdownMenuRadioItem.displayName = DropdownMenuPrimitive.RadioItem.displayName 138 | 139 | const DropdownMenuLabel = React.forwardRef< 140 | React.ElementRef, 141 | React.ComponentPropsWithoutRef & { 142 | inset?: boolean 143 | } 144 | >(({ className, inset, ...props }, ref) => ( 145 | 154 | )) 155 | DropdownMenuLabel.displayName = DropdownMenuPrimitive.Label.displayName 156 | 157 | const DropdownMenuSeparator = React.forwardRef< 158 | React.ElementRef, 159 | React.ComponentPropsWithoutRef 160 | >(({ className, ...props }, ref) => ( 161 | 166 | )) 167 | DropdownMenuSeparator.displayName = DropdownMenuPrimitive.Separator.displayName 168 | 169 | const DropdownMenuShortcut = ({ 170 | className, 171 | ...props 172 | }: React.HTMLAttributes) => { 173 | return ( 174 | 181 | ) 182 | } 183 | DropdownMenuShortcut.displayName = "DropdownMenuShortcut" 184 | 185 | export { 186 | DropdownMenu, 187 | DropdownMenuTrigger, 188 | DropdownMenuContent, 189 | DropdownMenuItem, 190 | DropdownMenuCheckboxItem, 191 | DropdownMenuRadioItem, 192 | DropdownMenuLabel, 193 | DropdownMenuSeparator, 194 | DropdownMenuShortcut, 195 | DropdownMenuGroup, 196 | DropdownMenuPortal, 197 | DropdownMenuSub, 198 | DropdownMenuSubContent, 199 | DropdownMenuSubTrigger, 200 | DropdownMenuRadioGroup, 201 | } 202 | -------------------------------------------------------------------------------- /client/src/components/base/nav/NavTitle.tsx: -------------------------------------------------------------------------------- 1 | import {Link} from "wouter"; 2 | 3 | export function NavTitle() { 4 | return ( 5 |
6 |
7 | 8 |

Home

9 | 10 |
11 |
12 | ); 13 | } 14 | -------------------------------------------------------------------------------- /client/src/components/base/nav/Navbar.tsx: -------------------------------------------------------------------------------- 1 | import {Avatar} from "@/components/base/avatar/Avatar"; 2 | import {authStore, logout} from "@/auth/authStore"; 3 | import {NavTitle} from "@/components/base/nav/NavTitle"; 4 | import { 5 | DropdownMenu, 6 | DropdownMenuContent, 7 | DropdownMenuTrigger, 8 | DropdownMenuLabel, 9 | DropdownMenuSeparator, DropdownMenuItem 10 | } from "@/components/base/menu/DropdownMenu"; 11 | import {cn} from "@/lib/utils"; 12 | 13 | export function Navbar({className}: {className?: string}) { 14 | return ( 15 | 19 | ); 20 | } 21 | 22 | function ProfileAvatar() { 23 | const [displayName] = authStore.use('displayName'); 24 | const src = authStore.use('photoUrl')[0] ?? undefined; 25 | 26 | return ( 27 | 28 | 29 | 30 | {displayName?.slice(0, 1) ?? '?'} 31 | 32 | 33 | 34 | My Account 35 | Settings 36 | Billing 37 | 38 | Sign Out 39 | 40 | 41 | ) 42 | } 43 | -------------------------------------------------------------------------------- /client/src/components/base/nav/Sidebar.tsx: -------------------------------------------------------------------------------- 1 | import {Link, useLocation} from "wouter"; 2 | import {cn} from "@/lib/utils"; 3 | import {IconType} from "react-icons"; 4 | import {FaCog, FaHome, FaPlayCircle} from "react-icons/all"; 5 | 6 | type NavLinkProps = {name: string, to: string, icon: IconType}; 7 | const navigation = [ 8 | {name: 'Dashboard', to: '/', icon: FaHome}, 9 | ] as const satisfies ReadonlyArray; 10 | 11 | const docs = [ 12 | {name: 'Getting Started', to: '/getting-started', icon: FaPlayCircle}, 13 | ] as const satisfies ReadonlyArray; 14 | 15 | export function Sidebar() { 16 | return ( 17 |
18 |
19 | Created by Acorn1010 24 |
25 | 42 |
43 | ); 44 | } 45 | 46 | function NavLink({name, to, icon: Icon}: NavLinkProps) { 47 | const location = useLocation()[0]; 48 | const isCurrent = location === to; 49 | return ( 50 |
  • 51 | 60 |
    • 64 | ); 65 | } 66 | 67 | function BottomNavLink({name, to, icon: Icon}: NavLinkProps) { 68 | const location = useLocation()[0]; 69 | const isCurrent = location === to; 70 | return ( 71 |
  • 72 | 81 |
    • 85 | ); 86 | } 87 | 88 | function Background() { 89 | return ( 90 | <> 91 |
    92 | 93 | ); 94 | } 95 | -------------------------------------------------------------------------------- /client/src/components/base/styles.ts: -------------------------------------------------------------------------------- 1 | import {cn} from "@/lib/utils"; 2 | 3 | /** Adds a focus ring to the class. Used for elements that can be focused (e.g. Button). */ 4 | export const focusStyle = cn` 5 | dark:focus:ring-offset-slate-900 6 | dark:focus:ring-slate-400 7 | 8 | focus:outline-none 9 | focus:ring-2 10 | focus:ring-offset-2 11 | focus:ring-slate-400 12 | `; 13 | -------------------------------------------------------------------------------- /client/src/components/buttons/CopyButton.tsx: -------------------------------------------------------------------------------- 1 | import {cn} from "@/lib/utils"; 2 | import {IconButton} from "@/components/base/buttons/IconButton"; 3 | import {FaCheck, FaCopy} from "react-icons/all"; 4 | import {useState} from "react"; 5 | 6 | /** A button that allows copying `value` to the clipboard. */ 7 | export function CopyButton({className, value}: {className?: string, value: string}) { 8 | const [isCopied, setIsCopied] = useState(false); 9 | 10 | const CopyIcon = isCopied ? FaCheck : FaCopy; 11 | return ( 12 | { 16 | navigator.clipboard.writeText(value).then(() => { 17 | setIsCopied(true); 18 | setTimeout(() => setIsCopied(false), 1_500); 19 | }); 20 | }} /> 21 | ) 22 | } 23 | -------------------------------------------------------------------------------- /client/src/components/cards/DashboardCard.tsx: -------------------------------------------------------------------------------- 1 | import {PropsWithChildren} from "react"; 2 | import {cn} from "@/lib/utils"; 3 | 4 | export function DashboardCard({children, className}: PropsWithChildren<{className?: string}>) { 5 | return ( 6 |
    7 | {children} 8 |
    9 | ); 10 | } 11 | -------------------------------------------------------------------------------- /client/src/components/inputs/ApiToken.tsx: -------------------------------------------------------------------------------- 1 | import {Input} from "@/components/base/inputs/Input"; 2 | import {useState} from "react"; 3 | import {FaEye, FaEyeSlash} from "react-icons/all"; 4 | import {IconButton} from "@/components/base/buttons/IconButton"; 5 | import {CopyButton} from "@/components/buttons/CopyButton"; 6 | import {useApiToken} from "@/auth/useApiToken"; 7 | 8 | export function ApiToken() { 9 | const [isHidden, setIsHidden] = useState(true); 10 | const token = useApiToken(); 11 | 12 | const EyeIcon = isHidden ? FaEyeSlash : FaEye; 13 | return ( 14 |
    15 |

    API Token

    16 |
    17 | 18 | setIsHidden(prevState => !prevState)} /> 22 |
    23 | 24 |
    25 | ) 26 | } 27 | 28 | -------------------------------------------------------------------------------- /client/src/components/tables/SortedTable.tsx: -------------------------------------------------------------------------------- 1 | import {useState} from "react"; 2 | import {cn} from "@/lib/utils"; 3 | import {FaChevronDown, FaChevronUp} from "react-icons/all"; 4 | import {localizeNumber} from "@/lib/StringUtils"; 5 | 6 | const RESULTS_PER_PAGE = 100; 7 | 8 | type SortedTableProps = { 9 | /** The rows in the table. */ 10 | rows: T[], 11 | 12 | /** If provided, this is the default sort that will be applied to the table. */ 13 | defaultSort?: {column: keyof T, dir: 'asc' | 'desc'}, 14 | }; 15 | 16 | export function SortableTable(props: SortedTableProps) { 17 | const {rows: unsortedRows, defaultSort} = props; 18 | const [sortOrDefault, setSort] = 19 | useState<{column: keyof T, dir: 'asc' | 'desc'} | null>(defaultSort ?? null); 20 | 21 | const [page] = useState(0); // TODO(acorn1010): Add support for pagination 22 | const keys = Object.keys(unsortedRows[0] ?? {}) as (keyof T)[]; 23 | const sort = sortOrDefault ?? {column: keys[0], dir: 'asc'}; 24 | 25 | // Sort the rows to display and slice on what data to show 26 | const rows = unsortedRows.sort((a, b) => { 27 | const aVal = a[sort.column]; 28 | const bVal = b[sort.column]; 29 | const direction = sort.dir === 'asc' ? 1 : -1; 30 | if (typeof aVal === 'string' && typeof bVal === 'string') { 31 | return direction * aVal.localeCompare(bVal); 32 | } else if (typeof aVal === 'number' && typeof bVal === 'number') { 33 | return direction * (aVal - bVal); 34 | } 35 | // TODO(acorn1010): Allow other types 36 | return 0; 37 | }).slice(page * RESULTS_PER_PAGE, (page + 1) * RESULTS_PER_PAGE); 38 | 39 | return ( 40 |
    41 |
    42 |
    43 | 44 | 45 | 46 | {keys.map(key => ( 47 | 71 | ))} 72 | 73 | 74 | 75 | {rows.map((row, idx) => ( 76 | 79 | {Object.entries(row).map(([column, value]) => 80 | )} 83 | 84 | ))} 85 | 86 |
    48 | 70 |
    81 | {typeof value === 'number' ? localizeNumber(value) : value} 82 |
    87 |
    88 |
    89 |
    90 | ); 91 | } 92 | -------------------------------------------------------------------------------- /client/src/index.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | /* We do _not_ want @import to show up before tailwind, or it messes up initial load. */ 6 | /*noinspection CssInvalidImport*/ 7 | @import url('https://fonts.googleapis.com/css2?family=Roboto:ital,wght@0,100;0,300;0,400;0,500;0,700;0,900;1,100;1,300;1,400;1,500;1,700;1,900&display=swap'); 8 | /*noinspection CssInvalidImport*/ 9 | @import url('https://fonts.googleapis.com/css2?family=Roboto%20Mono:ital,wght@0,100;0,300;0,400;0,500;0,700;0,900;1,100;1,300;1,400;1,500;1,700;1,900&display=swap'); 10 | 11 | @layer utilities { 12 | .flex-center { 13 | @apply flex justify-center items-center 14 | } 15 | 16 | .text-shadow-none { 17 | text-shadow: none; 18 | } 19 | } 20 | 21 | html { 22 | --font-mono: 'Roboto Mono'; 23 | --font-sans: 'Roboto'; 24 | } 25 | -------------------------------------------------------------------------------- /client/src/lib/StringUtils.ts: -------------------------------------------------------------------------------- 1 | 2 | /** Localizes a number "value" to its comma-separated value for the user's locale. */ 3 | export function localizeNumber(number: number): string { 4 | return number.toLocaleString(navigator.language ?? 'en-US'); 5 | } 6 | -------------------------------------------------------------------------------- /client/src/lib/utils.ts: -------------------------------------------------------------------------------- 1 | import { ClassValue, clsx } from "clsx" 2 | import { twMerge } from "tailwind-merge" 3 | 4 | /** Wrapper around twMerge and clsx. Use this instead. */ 5 | export function cn(...inputs: ClassValue[]) { 6 | return twMerge(clsx(inputs)) 7 | } 8 | -------------------------------------------------------------------------------- /client/src/main.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import ReactDOM from 'react-dom/client' 3 | import {App} from './App' 4 | 5 | ReactDOM.createRoot(document.getElementById('root') as HTMLElement).render( 6 | 7 | ); 8 | -------------------------------------------------------------------------------- /client/src/pages/LoginPage.tsx: -------------------------------------------------------------------------------- 1 | import {Button} from "@/components/base/buttons/Button"; 2 | import {AuthProviderId, signInWithProvider} from "@/auth/authStore"; 3 | import {ReactElement} from "react"; 4 | import {FaGithub, FaGoogle} from "react-icons/all"; 5 | 6 | const PROVIDER_BUTTONS = { 7 | google: {name: 'Google', logo: FaGoogle}, 8 | github: {name: 'GitHub', logo: FaGithub}, 9 | } satisfies {[provider in AuthProviderId]: {name: string, logo: (props: {className?: string}) => ReactElement}}; 10 | 11 | export default function LoginPage() { 12 | return ( 13 |
    14 |

    Log In

    15 | 16 | 17 |
    18 | ); 19 | } 20 | 21 | function LoginButton({provider}: {provider: keyof typeof PROVIDER_BUTTONS}) { 22 | const {name, logo: Logo} = PROVIDER_BUTTONS[provider]; 23 | return ( 24 | 27 | ); 28 | } 29 | -------------------------------------------------------------------------------- /client/src/pages/LogoutPage.tsx: -------------------------------------------------------------------------------- 1 | import {useEffect} from "react"; 2 | import {logout} from "@/auth/authStore"; 3 | import {useLocation} from "wouter"; 4 | 5 | export default function LogoutPage() { 6 | const [, navigate] = useLocation(); 7 | useEffect(() => { 8 | (async () => { 9 | await logout(); 10 | navigate('/'); 11 | })(); 12 | }, [navigate]); 13 | 14 | return

    Logging out...

    ; 15 | } 16 | -------------------------------------------------------------------------------- /client/src/pages/NotFoundPage.tsx: -------------------------------------------------------------------------------- 1 | import {Link} from "wouter"; 2 | 3 | export default function NotFoundPage() { 4 | return ( 5 |
    6 |

    404

    7 |

    Page not found

    8 |

    Sorry, we couldn’t find the page you’re looking for.

    9 |
    10 | 11 | Go back home 12 | 13 |
    14 |
    15 | ); 16 | } 17 | -------------------------------------------------------------------------------- /client/src/pages/dashboard/Dashboard.tsx: -------------------------------------------------------------------------------- 1 | import {authStore} from "@/auth/authStore"; 2 | import {PropsWithChildren} from "react"; 3 | import {Navbar} from "@/components/base/nav/Navbar"; 4 | import {Sidebar} from "@/components/base/nav/Sidebar"; 5 | import {Redirect, Switch} from "wouter"; 6 | import {Route} from "@/Route"; 7 | import DashboardHomePage from "@/pages/dashboard/DashboardHomePage"; 8 | 9 | export default function Dashboard() { 10 | return ( 11 | 12 | 13 | 14 | 15 | 16 | import('./GettingStartedPage')} /> 17 | import('./SettingsPage')} /> 18 | 19 | 20 | 21 | 22 | 23 | ); 24 | } 25 | 26 | /** Adds a Navbar / side panel to the dashboard. Included in every dashboard route. */ 27 | function DashboardContainer({children}: PropsWithChildren<{}>) { 28 | return ( 29 |
    30 | 31 |
    32 | 33 |
    {children}
    34 |
    35 |
    36 | ); 37 | } 38 | -------------------------------------------------------------------------------- /client/src/pages/dashboard/DashboardHomePage.tsx: -------------------------------------------------------------------------------- 1 | import {FaPlus} from "react-icons/all"; 2 | import {Link} from "wouter"; 3 | import {poll} from "@/api/call"; 4 | import {isLoaded} from "@/state/isLoaded"; 5 | import {Skeleton} from "@/components/base/loading/Skeleton"; 6 | import {SortableTable} from "@/components/tables/SortedTable"; 7 | 8 | export default function DashboardHomePage() { 9 | const renderCounts = poll.use('getMonthlyRenderCounts'); 10 | 11 | if (!isLoaded(renderCounts)) { 12 | return 13 | } 14 | 15 | return renderCounts ? : ; 16 | } 17 | 18 | function MonthlyRenderTable({renderCounts}: {renderCounts: {month: string, renderCount: number}[]}) { 19 | return ( 20 |
    21 |
    22 |
    23 |

    Renders by Month

    24 |

    25 | A list of the total page renders for each month. 26 |

    27 |
    28 |
    29 | ({'Month': month, 'Render Count': renderCount}) 32 | )} 33 | defaultSort={{column: 'Month', dir: 'desc'}} 34 | /> 35 |
    36 | ); 37 | } 38 | 39 | function EmptyDashboard() { 40 | return ( 41 |
    42 | 57 |

    No recent renders

    58 |

    Get started by adding a Cloudflare worker render script for your site.

    59 |
    60 | 64 |
    68 |
    69 | ); 70 | } 71 | -------------------------------------------------------------------------------- /client/src/pages/dashboard/GettingStartedPage.tsx: -------------------------------------------------------------------------------- 1 | import {FaAngular, FaReact, FaVuejs} from "react-icons/all"; 2 | import {DashboardCard} from "@/components/cards/DashboardCard"; 3 | import cloudflareWorker from './cloudflare_worker.js?raw'; 4 | import {CodeBlock} from "@/components/base/blocks/CodeBlock"; 5 | import {useApiToken} from "@/auth/useApiToken"; 6 | 7 | export default function GettingStartedPage() { 8 | const token = useApiToken(); 9 | const tokenText = token && (the script already has your API key); 10 | const code = cloudflareWorker.replace("API_KEY = '';", `API_KEY = '${token}';`); 11 | return ( 12 | 13 |

    How to Render Your Single Page App for SEO

    14 |
    15 | 16 |
    17 |

    Boost the SEO of your JavaScript Single Page App (SPA) by rendering your content to HTML. This allows search engines to index your website, making it more visible to users. Follow these simple steps to pre-render your SPA using a Cloudflare Worker.

    18 | 19 |

    Requirements

    20 |
      21 |
    • Assumes you're using Cloudflare's CDN.
      • 22 |
    23 | 24 |

    Steps

    25 |
      26 |
    1. Open your CloudFlare dashboard.
      2. 27 |
    2. Go to {'Workers > Overview'}.
      3. 28 |
    3. Click on .
      4. 29 |
    4. Give your service a name you'll remember (e.g. ), then click .
      5. 30 |
    5. Click on Triggers then .
      6. 31 |
    6. Add your route (e.g. example.com/*)
      7. 32 |
    7. Now click on in the top-right.
      8. 33 |
    8. 34 | Paste the following worker script{tokenText}: 35 | :nth-child(5)]:!text-transparent [&>:nth-child(5)]:text-shadow-none', 40 | }} 41 | code={code} 42 | language='javascript'> 43 | 44 |
      9. 45 |
    9. Finally, click .
      10. 46 |
    10. Test that you're being served rendered HTML by changing your UserAgent to or : 47 |
        48 | 49 | 50 |
      51 |
      11. 52 |
    53 |     54 | ); 55 | } 56 | 57 | function FakeCloudflareButton({text}: {text: string}) { 58 | return {text}; 59 | } 60 | 61 | function StringText({text}: {text: string}) { 62 | return {text}; 63 | } 64 | 65 | function CommandListItem({text}: {text: string}) { 66 | return
  • {text}
    • 67 | } 68 | -------------------------------------------------------------------------------- /client/src/pages/dashboard/SettingsPage.tsx: -------------------------------------------------------------------------------- 1 | import {call, poll} from "@/api/call"; 2 | import {Button} from "@/components/base/buttons/Button"; 3 | import {ApiToken} from "@/components/inputs/ApiToken"; 4 | import {DashboardCard} from "@/components/cards/DashboardCard"; 5 | 6 | export default function SettingsPage() { 7 | return ( 8 | 9 |

    Settings

    10 |
    11 | 12 | 18 |
    19 |     20 | ); 21 | } 22 | -------------------------------------------------------------------------------- /client/src/pages/dashboard/cloudflare_worker.js: -------------------------------------------------------------------------------- 1 | // API key for requests. 2 | const API_KEY = ''; 3 | 4 | // That's it! You shouldn't need to change anything below here unless you really want to. 5 | 6 | // These are the user agents that the worker will look for to 7 | // initiate prerendering of the site. 8 | const BOT_AGENTS = [ 9 | 'applebot', 10 | 'baiduspider', 11 | 'bingbot', 12 | 'bitlybot', 13 | 'bitrix link preview', 14 | 'chrome-lighthouse', 15 | 'developers.google.com/+/web/snippet', 16 | 'discordbot', 17 | 'embedly', 18 | 'facebookexternalhit', 19 | 'flipboard', 20 | 'google page speed', 21 | 'googlebot', 22 | 'linkedinbot', 23 | 'lynx/', 24 | 'nuzzel', 25 | 'outbrain', 26 | 'pinterest/0.', 27 | 'pinterestbot', 28 | 'quora link preview', 29 | 'qwantify', 30 | 'redditbot', 31 | 'rogerbot', 32 | 'showyoubot', 33 | 'skypeuripreview', 34 | 'slackbot', 35 | 'telegrambot', 36 | 'tumblr', 37 | 'twitterbot', 38 | 'vkshare', 39 | 'w3c_validator', 40 | 'whatsapp', 41 | 'xing-contenttabreceiver', 42 | 'yahoo! slurp', 43 | 'yandex', 44 | ]; 45 | 46 | // These are the extensions that the worker will skip prerendering 47 | // even if any other conditions pass. 48 | // NOTE: You may want to add .json here 49 | const IGNORE_EXTENSIONS = [ 50 | '.ai', 51 | '.avi', 52 | '.avif', 53 | '.css', 54 | '.dat', 55 | '.dmg', 56 | '.doc', 57 | '.doc', 58 | '.eot', 59 | '.exe', 60 | '.flv', 61 | '.gif', 62 | '.ico', 63 | '.iso', 64 | '.jpeg', 65 | '.jpg', 66 | '.js', 67 | '.less', 68 | '.m4a', 69 | '.m4v', 70 | '.mov', 71 | '.mp3', 72 | '.mp4', 73 | '.mpeg', 74 | '.mpg', 75 | '.ogg', 76 | '.opus', 77 | '.otf', 78 | '.pdf', 79 | '.png', 80 | '.ppt', 81 | '.psd', 82 | '.rar', 83 | '.rss', 84 | '.svg', 85 | '.swf', 86 | '.tif', 87 | '.torrent', 88 | '.ttc', 89 | '.ttf', 90 | '.txt', 91 | '.wav', 92 | '.webm', 93 | '.webmanifest', 94 | '.webp', 95 | '.wmv', 96 | '.woff', 97 | '.woff2', 98 | '.xls', 99 | '.xml', 100 | '.zip', 101 | ]; 102 | 103 | /** 104 | * This attaches the event listener that gets invoked when CloudFlare receives 105 | * a request. 106 | */ 107 | addEventListener('fetch', event => { 108 | const {request} = event; 109 | const requestUserAgent = (request.headers.get('User-Agent') || '').toLowerCase(); 110 | const xPrerender = request.headers.get('X-Prerender'); 111 | const url = new URL(request.url); 112 | const pathName = url.pathname.toLowerCase(); 113 | const ext = pathName.substring(pathName.lastIndexOf('.') || pathName.length); 114 | 115 | if ( 116 | !xPrerender 117 | && BOT_AGENTS.some(e => requestUserAgent.includes(e)) 118 | && !IGNORE_EXTENSIONS.some(e => e === ext) 119 | && /^https?:\/\//.test(request.url) 120 | ) { 121 | event.respondWith(prerenderRequest(request)); 122 | } 123 | }); 124 | 125 | /** 126 | * Function to request the prerendered version of a request. 127 | * 128 | * @param {Request} request - The request received by CloudFlare 129 | * @returns {Promise} 130 | */ 131 | function prerenderRequest(request) { 132 | const {url, headers} = request; 133 | const base = 'https://api.rendermy.site'; 134 | const prerenderUrl = `${base}/${url}`; 135 | 136 | const headersToSend = new Headers(headers); 137 | headersToSend.set('X-Prerender-Token', API_KEY); 138 | 139 | const prerenderRequest = new Request(prerenderUrl, { 140 | headers: headersToSend, 141 | redirect: 'manual', 142 | }); 143 | 144 | // Cache for 1 day 145 | // See: https://developers.cloudflare.com/workers/runtime-apis/request/#requestinitcfproperties 146 | return fetch(prerenderRequest, {cf: {cacheTtl: 24 * 60 * 60 }}); 147 | } 148 | -------------------------------------------------------------------------------- /client/src/services/firebase.ts: -------------------------------------------------------------------------------- 1 | import {FirebaseOptions, initializeApp} from "firebase/app"; 2 | 3 | const firebaseConfig: FirebaseOptions = { 4 | apiKey: "AIzaSyCDDmmymNeYql3uZVC1kX7F6sr2ZbG5SKk", 5 | authDomain: "render-1010.firebaseapp.com", 6 | projectId: "render-1010", 7 | storageBucket: "render-1010.appspot.com", 8 | messagingSenderId: "263893907468", 9 | appId: "1:263893907468:web:44d02403da6f19e06ed629", 10 | measurementId: "G-H4DG1NC8X2", 11 | }; 12 | export const app = initializeApp(firebaseConfig, {automaticDataCollectionEnabled: false}); 13 | -------------------------------------------------------------------------------- /client/src/state/createGlobalStore.ts: -------------------------------------------------------------------------------- 1 | import {SetStateAction, useCallback} from 'react'; 2 | import {create} from "zustand"; 3 | 4 | export type EqualityFn = (left: T | null | undefined, right: T | null | undefined) => boolean; 5 | 6 | // eslint-disable-next-line @typescript-eslint/ban-types 7 | const isFunction = (fn: unknown): fn is Function => (typeof fn === 'function'); 8 | 9 | /** 10 | * Create a global state 11 | * 12 | * It returns a set of functions 13 | * - `use`: Works like React.useState. "Registers" the component as a listener on that key 14 | * - `get`: retrieves a key without a re-render 15 | * - `set`: sets a key. Causes re-renders on any listeners 16 | * - `getAll`: retrieves the entire state (all keys) as an object without a re-render 17 | * - `reset`: resets the state back to its initial value 18 | * 19 | * @example 20 | * import { createStore } from 'create-store'; 21 | * 22 | * const store = createStore({ count: 0 }); 23 | * 24 | * const Component = () => { 25 | * const [count, setCount] = store.use('count'); 26 | * ... 27 | * }; 28 | */ 29 | export const createGlobalStore = (initialState: State) => { 30 | // NOTE: Not using structuredClone because browser support only goes about 2 years back. 31 | const store = create(() => deepClone(initialState)); 32 | 33 | const setter = (key: T, value: SetStateAction) => { 34 | if (isFunction(value)) { 35 | store.setState(prevValue => ({[key]: value(prevValue[key])} as unknown as Partial)); 36 | } else { 37 | store.setState({[key]: value} as unknown as Partial); 38 | } 39 | }; 40 | return { 41 | /** Works like React.useState. "Registers" the component as a listener on that key. */ 42 | use( 43 | key: K, 44 | defaultValue?: State[K], 45 | equalityFn?: EqualityFn): [State[K], (value: SetStateAction) => void] { 46 | // If state isn't defined for a given defaultValue, set it. 47 | if (defaultValue !== undefined && !(key in store.getState())) { 48 | setter(key, defaultValue); 49 | } 50 | const result = store(state => state[key], equalityFn); 51 | // eslint-disable-next-line react-hooks/rules-of-hooks 52 | const keySetter = useCallback((value: SetStateAction) => setter(key, value), [key]); 53 | return [result, keySetter]; 54 | }, 55 | 56 | /** Listens on the entire state, causing a re-render when anything in the state changes. */ 57 | useAll: () => store(state => state), 58 | 59 | /** Deletes a `key` from state, causing a re-render for anything listening. */ 60 | delete(key: K) { 61 | store.setState(prevState => { 62 | const {[key]: _, ...rest} = prevState; 63 | return rest as Partial; 64 | }, true); 65 | }, 66 | 67 | /** Retrieves the current `key` value. Does _not_ listen on state changes (meaning no re-renders). */ 68 | get(key: K) { 69 | return store.getState()[key]; 70 | }, 71 | 72 | /** Retrieves the entire state. Does _not_ listen on state changes (meaning no re-renders). */ 73 | getAll: () => store.getState(), 74 | 75 | /** Returns `true` if `key` is in the state. */ 76 | has(key: K) { 77 | return key in store.getState(); 78 | }, 79 | 80 | /** Sets a `key`, triggering a re-render for all listeners. */ 81 | set: setter, 82 | 83 | /** Sets the entire state, removing any keys that aren't present in `state`. */ 84 | setAll: (state: State) => store.setState(state, true), 85 | 86 | /** Updates the keys in `state`, leaving any keys / values not in `state` unchanged. */ 87 | update: (state: Partial) => store.setState(state, false), 88 | 89 | /** Resets the entire state back to its initial state when the store was created. */ 90 | reset: () => store.setState(deepClone(initialState), true), 91 | }; 92 | }; 93 | 94 | /** 95 | * Returns a wrapped `store` that can't be modified. Useful when you want to 96 | * control who is able to write to a store. 97 | */ 98 | export function createReadonlyStore>( 99 | store: T) { 100 | type State = ReturnType; 101 | return { 102 | get: store.get, 103 | getAll: store.getAll, 104 | use: (key: K, equalityFn?: EqualityFn) => 105 | (store.use as any)(key, undefined, equalityFn)[0] as State[K] | undefined | null, 106 | useAll: store.useAll, 107 | }; 108 | } 109 | 110 | /** 111 | * Deeply copies objects. Borrowed from just-clone, but with some nicer types. 112 | * See: https://github.com/angus-c/just/blob/master/packages/collection-clone/index.cjs 113 | */ 114 | function deepClone(obj: T): T { 115 | let result = obj; 116 | const type = {}.toString.call(obj).slice(8, -1); 117 | if (type === 'Set') { 118 | return new Set([...obj as Set].map(value => deepClone(value))) as any; 119 | } 120 | if (type === 'Map') { 121 | return new Map([...obj as Set].map(kv => [deepClone(kv[0]), deepClone(kv[1])])) as any; 122 | } 123 | if (type === 'Date') { 124 | return new Date((obj as Date).getTime()) as any; 125 | } 126 | if (type === 'RegExp') { 127 | return RegExp((obj as RegExp).source as string, getRegExpFlags(obj as RegExp)) as any; 128 | } 129 | if (type === 'Array' || type === 'Object') { 130 | result = Array.isArray(obj) ? [] : {} as any; 131 | for (const key in obj) { 132 | // include prototype properties 133 | result[key] = deepClone(obj[key]); 134 | } 135 | } 136 | // primitives and non-supported objects (e.g. functions) land here 137 | return result; 138 | } 139 | 140 | function getRegExpFlags(regExp: RegExp): string { 141 | if ((typeof regExp.source as any).flags === 'string') { 142 | return (regExp.source as any).flags; 143 | } else { 144 | const flags = []; 145 | regExp.global && flags.push('g'); 146 | regExp.ignoreCase && flags.push('i'); 147 | regExp.multiline && flags.push('m'); 148 | regExp.sticky && flags.push('y'); 149 | regExp.unicode && flags.push('u'); 150 | return flags.join(''); 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /client/src/state/isLoaded.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Returns `true` if `value` is loaded (meaning it's T or null). `undefined` is used to indicate 3 | * that the location is still being loaded by the database, while `null` means the data doesn't 4 | * exist within the database. 5 | */ 6 | export function isLoaded(value: T | null | undefined): value is T { 7 | return value !== undefined; 8 | } 9 | -------------------------------------------------------------------------------- /client/src/state/useLocalStorage.ts: -------------------------------------------------------------------------------- 1 | import {useCallback, useEffect} from "react"; 2 | import {createGlobalStore} from "@/state/createGlobalStore"; 3 | 4 | const store = createGlobalStore({} as {[key: string]: any}); 5 | 6 | /** 7 | * Listens on changes to a `key` in localStorage. Returns the value of the key 8 | * @param key the key to listen to 9 | * @param initialValue the initial value to use if the key doesn't exist in localStorage 10 | */ 11 | export function useLocalStorage(key: string, initialValue?: T) 12 | : [T, (value: T | ((prevValue: T) => T)) => void] { 13 | const [result, setResult] = store.use(key); 14 | const storedValue = result ?? getLocalStorage(key) ?? (initialValue ?? null); 15 | 16 | // Set store if we had localStorage or an initial value that hasn't been set yet. 17 | useEffect(() => { 18 | if (result === undefined && storedValue !== result) { 19 | setResult(storedValue); 20 | } 21 | }, [result, storedValue, setResult]); 22 | 23 | const setResultWrapper = useCallback((newValue: T | ((prevValue: T) => T)) => { 24 | // Allow value to be a function so we have the same API as useState 25 | const valueToStore = newValue instanceof Function ? newValue(store.get(key)) : newValue; 26 | // Save state 27 | setLocalStorage(key, valueToStore); 28 | }, [key]); 29 | 30 | return [storedValue, setResultWrapper]; 31 | } 32 | 33 | /** 34 | * Retrieves a value from local storage. Doesn't trigger any React state changes. 35 | * @param key the key to get 36 | */ 37 | export function getLocalStorage(key: string): T | null { 38 | try { 39 | const item = window.localStorage.getItem(key); 40 | return item ? JSON.parse(item) : null; 41 | } catch (error) { 42 | console.error('Failed to get from local storage.', key, error); 43 | return null; 44 | } 45 | } 46 | 47 | /** 48 | * Sets a value in local storage. 49 | * @param key the key to set 50 | * @param value the value to set at location "key" 51 | */ 52 | export function setLocalStorage(key: string, value: T) { 53 | try { 54 | store.set(key, value); 55 | window.localStorage.setItem(key, JSON.stringify(value)); 56 | } catch (error) { 57 | console.error('Failed to set local storage.', key, value, error); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /client/src/vite-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /client/tailwind.config.cjs: -------------------------------------------------------------------------------- 1 | const { fontFamily } = require("tailwindcss/defaultTheme"); 2 | 3 | /** @type {import('tailwindcss').Config} */ 4 | module.exports = { 5 | content: [ 6 | "index.html", 7 | "./src/**/*.{js,jsx,ts,tsx}", 8 | ], 9 | darkMode: ["class", '[data-theme="dark"]'], 10 | 11 | theme: { 12 | extend: { 13 | colors: { 14 | lighten: 'rgba(255, 255, 255, .1)', 15 | darken: 'rgba(0, 0, 0, .1)', 16 | }, 17 | fontFamily: { 18 | mono: ["var(--font-mono)", ...fontFamily.mono], 19 | sans: ["var(--font-sans)", ...fontFamily.sans], 20 | }, 21 | keyframes: { 22 | "accordion-down": { 23 | from: { height: 0 }, 24 | to: { height: "var(--radix-accordion-content-height)" }, 25 | }, 26 | "accordion-up": { 27 | from: { height: "var(--radix-accordion-content-height)" }, 28 | to: { height: 0 }, 29 | }, 30 | }, 31 | animation: { 32 | "accordion-down": "accordion-down 0.2s ease-out", 33 | "accordion-up": "accordion-up 0.2s ease-out", 34 | }, 35 | }, 36 | }, 37 | plugins: [require("tailwindcss-animate")], 38 | }; 39 | -------------------------------------------------------------------------------- /client/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "baseUrl": ".", 4 | "paths": { 5 | "@/*": ["./src/*"], 6 | "@shared/*": ["../shared/src/*"] 7 | }, 8 | "target": "ESNext", 9 | "useDefineForClassFields": true, 10 | "lib": ["DOM", "DOM.Iterable", "ESNext"], 11 | "allowJs": false, 12 | "skipLibCheck": true, 13 | "esModuleInterop": false, 14 | "allowSyntheticDefaultImports": true, 15 | "strict": true, 16 | "forceConsistentCasingInFileNames": true, 17 | "module": "ESNext", 18 | "moduleResolution": "Node", 19 | "resolveJsonModule": true, 20 | "isolatedModules": true, 21 | "noEmit": true, 22 | "jsx": "react-jsx" 23 | }, 24 | "include": ["src", "../shared/src"], 25 | "references": [{ "path": "./tsconfig.node.json" }] 26 | } 27 | -------------------------------------------------------------------------------- /client/tsconfig.node.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "composite": true, 4 | "module": "ESNext", 5 | "moduleResolution": "Node", 6 | "allowSyntheticDefaultImports": true 7 | }, 8 | "include": ["vite.config.ts"] 9 | } 10 | -------------------------------------------------------------------------------- /client/vite.config.ts: -------------------------------------------------------------------------------- 1 | import * as path from 'path'; 2 | import { defineConfig } from 'vite' 3 | import react from '@vitejs/plugin-react' 4 | 5 | // https://vitejs.dev/config/ 6 | export default defineConfig({ 7 | plugins: [react()], 8 | resolve: { 9 | alias: [ 10 | {find: "@", replacement: path.resolve(__dirname, "src")}, 11 | {find: "@shared", replacement: path.resolve(__dirname, "../shared/src")}, 12 | ], 13 | }, 14 | esbuild: { 15 | /** default is "eof", which takes up ~15% in the js files uncompressed (2% compressed). */ 16 | legalComments: 'external', 17 | }, 18 | }); 19 | -------------------------------------------------------------------------------- /hcloud-microos-snapshots.pkr.hcl: -------------------------------------------------------------------------------- 1 | /* 2 | * Creates a MicroOS snapshot for Kube-Hetzner 3 | */ 4 | 5 | variable "hcloud_token" { 6 | type = string 7 | default = env("HCLOUD_TOKEN") 8 | sensitive = true 9 | } 10 | 11 | # We download the OpenSUSE MicroOS x86 image from an automatically selected mirror. In case it somehow does not work for you (you get a 403), you can try other mirrors. 12 | # You can find a working mirror at https://download.opensuse.org/tumbleweed/appliances/openSUSE-MicroOS.x86_64-OpenStack-Cloud.qcow2.mirrorlist 13 | variable "opensuse_microos_x86_mirror_link" { 14 | type = string 15 | default = "https://ftp.gwdg.de/pub/opensuse/repositories/devel:/kubic:/images/openSUSE_Tumbleweed/openSUSE-MicroOS.x86_64-OpenStack-Cloud.qcow2" 16 | } 17 | 18 | # We download the OpenSUSE MicroOS ARM image from an automatically selected mirror. In case it somehow does not work for you (you get a 403), you can try other mirrors. 19 | # You can find a working mirror at https://download.opensuse.org/ports/aarch64/tumbleweed/appliances/openSUSE-MicroOS.aarch64-OpenStack-Cloud.qcow2.mirrorlist 20 | variable "opensuse_microos_arm_mirror_link" { 21 | type = string 22 | default = "https://ftp.gwdg.de/pub/opensuse/ports/aarch64/tumbleweed/appliances/openSUSE-MicroOS.aarch64-OpenStack-Cloud.qcow2" 23 | } 24 | 25 | # If you need to add other packages to the OS, do it here in the default value, like ["vim", "curl", "wget"] 26 | # When looking for packages, you need to search for OpenSUSE Tumbleweed packages, as MicroOS is based on Tumbleweed. 27 | variable "packages_to_install" { 28 | type = list(string) 29 | default = [] 30 | } 31 | 32 | locals { 33 | needed_packages = join(" ", concat(["restorecond policycoreutils policycoreutils-python-utils setools-console bind-utils wireguard-tools open-iscsi nfs-client xfsprogs cryptsetup lvm2 git cifs-utils"], var.packages_to_install)) 34 | 35 | # Add local variables for inline shell commands 36 | download_image = "wget --timeout=5 --waitretry=5 --tries=5 --retry-connrefused --inet4-only " 37 | 38 | write_image = <<-EOT 39 | set -ex 40 | echo 'MicroOS image loaded, writing to disk... ' 41 | qemu-img convert -p -f qcow2 -O host_device $(ls -a | grep -ie '^opensuse.*microos.*qcow2$') /dev/sda 42 | echo 'done. Rebooting...' 43 | sleep 1 && udevadm settle && reboot 44 | EOT 45 | 46 | install_packages = <<-EOT 47 | set -ex 48 | echo "First reboot successful, installing needed packages..." 49 | transactional-update shell <<< "setenforce 0" 50 | transactional-update --continue shell <<< "zypper --gpg-auto-import-keys install -y ${local.needed_packages}" 51 | transactional-update --continue shell <<< "rpm --import https://rpm-testing.rancher.io/public.key" 52 | transactional-update --continue shell <<< "zypper --no-gpg-checks --non-interactive install https://github.com/k3s-io/k3s-selinux/releases/download/v1.3.testing.4/k3s-selinux-1.3-4.sle.noarch.rpm" 53 | transactional-update --continue shell <<< "zypper addlock k3s-selinux" 54 | transactional-update --continue shell <<< "restorecon -Rv /etc/selinux/targeted/policy && restorecon -Rv /var/lib && setenforce 1" 55 | sleep 1 && udevadm settle && reboot 56 | EOT 57 | 58 | clean_up = <<-EOT 59 | set -ex 60 | echo "Second reboot successful, cleaning-up..." 61 | rm -rf /etc/ssh/ssh_host_* 62 | sleep 1 && udevadm settle 63 | EOT 64 | } 65 | 66 | # Source for the MicroOS x86 snapshot 67 | source "hcloud" "microos-x86-snapshot" { 68 | image = "ubuntu-22.04" 69 | rescue = "linux64" 70 | location = "fsn1" 71 | server_type = "cpx11" # disk size of >= 40GiB is needed to install the MicroOS image 72 | snapshot_labels = { 73 | microos-snapshot = "yes" 74 | creator = "kube-hetzner" 75 | } 76 | snapshot_name = "OpenSUSE MicroOS x86 by Kube-Hetzner" 77 | ssh_username = "root" 78 | token = var.hcloud_token 79 | } 80 | 81 | # Source for the MicroOS ARM snapshot 82 | source "hcloud" "microos-arm-snapshot" { 83 | image = "ubuntu-22.04" 84 | rescue = "linux64" 85 | location = "fsn1" 86 | server_type = "cax11" # disk size of >= 40GiB is needed to install the MicroOS image 87 | snapshot_labels = { 88 | microos-snapshot = "yes" 89 | creator = "kube-hetzner" 90 | } 91 | snapshot_name = "OpenSUSE MicroOS ARM by Kube-Hetzner" 92 | ssh_username = "root" 93 | token = var.hcloud_token 94 | } 95 | 96 | # Build the MicroOS x86 snapshot 97 | build { 98 | sources = ["source.hcloud.microos-x86-snapshot"] 99 | 100 | # Download the MicroOS x86 image 101 | provisioner "shell" { 102 | inline = ["${local.download_image}${var.opensuse_microos_x86_mirror_link}"] 103 | } 104 | 105 | # Write the MicroOS x86 image to disk 106 | provisioner "shell" { 107 | inline = [local.write_image] 108 | expect_disconnect = true 109 | } 110 | 111 | # Ensure connection to MicroOS x86 and do house-keeping 112 | provisioner "shell" { 113 | pause_before = "5s" 114 | inline = [local.install_packages] 115 | expect_disconnect = true 116 | } 117 | 118 | # Ensure connection to MicroOS x86 and do house-keeping 119 | provisioner "shell" { 120 | pause_before = "5s" 121 | inline = [local.clean_up] 122 | } 123 | } 124 | 125 | # Build the MicroOS ARM snapshot 126 | build { 127 | sources = ["source.hcloud.microos-arm-snapshot"] 128 | 129 | # Download the MicroOS ARM image 130 | provisioner "shell" { 131 | inline = ["${local.download_image}${var.opensuse_microos_arm_mirror_link}"] 132 | } 133 | 134 | # Write the MicroOS ARM image to disk 135 | provisioner "shell" { 136 | inline = [local.write_image] 137 | expect_disconnect = true 138 | } 139 | 140 | # Ensure connection to MicroOS ARM and do house-keeping 141 | provisioner "shell" { 142 | pause_before = "5s" 143 | inline = [local.install_packages] 144 | expect_disconnect = true 145 | } 146 | 147 | # Ensure connection to MicroOS ARM and do house-keeping 148 | provisioner "shell" { 149 | pause_before = "5s" 150 | inline = [local.clean_up] 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /kubernetes/.env.example: -------------------------------------------------------------------------------- 1 | # Your Hetzner API token for this project. Found at https://console.hetzner.cloud/projects/{projectId}/security/tokens 2 | HETZNER_TOKEN= 3 | 4 | # To get a list of k3s versions, run `hetzner-k3s releases` 5 | K3S_VERSION=v1.25.13+k3s1 6 | 7 | # Name of the cluster resources. Can be anything. A good name would be the name of your project 8 | # (e.g. "foony"). 9 | CLUSTER_NAME=mycluster 10 | 11 | # Physical location of the cluster. Valid locations: 12 | # Germany: nbg1, fsn1 13 | # Finland: hel1 14 | # US West: hil 15 | # US East: ash 16 | CLUSTER_LOCATION=ash 17 | 18 | # Number of master nodes to create. These nodes don't do work. They're in charge of keeping the cluster stable 19 | # and running. This should _always_ be an odd number. Increase to 3 if you want more uptime. Minimum of 1. 20 | # Each Master node costs about $8 / month. 21 | MASTERS_POOL_SIZE=1 22 | 23 | # Min number of worker nodes to create. Can be any positive integer. These are the nodes that run your code 24 | # (i.e. "do work"). Minimum of 0. 25 | # Each Worker node costs about $8 / month and gives 3 vCPU, 4 GB RAM, 20 TB bandwidth. 26 | WORKERS_POOL_SIZE_MIN=1 27 | 28 | # Max number of worker nodes to create. Can be any positive integer. These are the nodes that run your code 29 | # (i.e. "do work"). Minimum of 1. 30 | # Each Worker node costs about $8 / month and gives 3 vCPU, 4 GB RAM, 20 TB bandwidth. 31 | WORKERS_POOL_SIZE_MAX=2 32 | 33 | # Valid instance types. All instances come with 20 TB bandwidth. Only instances that are a "good deal" for 34 | # price / performance are listed here: 35 | # 36 | # cpx21 ($8/m, 3vCPU, 4GB RAM) 37 | # cpx41 ($27/m, 8vCPU, 16GB RAM) 38 | WORKERS_INSTANCE_TYPE=cpx21 39 | 40 | # Domain name for Rancher. Rancher is a GUI tool to help you manage your cluster. 41 | RANCHER_HOSTNAME=example.com 42 | 43 | # LetsEncrypt email address. This should be any valid email address under your control. 44 | LETSENCRYPT_EMAIL=letsencrypt@example.com 45 | 46 | # GCP Project ID for storing Google Container Registry images 47 | PROJECT_ID= 48 | 49 | # gcr.io secret. Used for deploying and accessing GCR containers from k3s. This should be the base64-encoded GCP 50 | # serviceAccount.json file. 51 | GCR_SECRET= 52 | 53 | # (optional) The password for Redis. Only needed if using the Redis service. 54 | REDIS_PASSWORD= 55 | -------------------------------------------------------------------------------- /kubernetes/README.md: -------------------------------------------------------------------------------- 1 | This folder contains all scripts necessary to create the initial cluster. 2 | 3 | To start a new cluster: 4 | 1. `cp .env.example .env` and fill in variables. 5 | 2. Run `./create.sh`. May need to run twice. 6 | 3. You're done! 7 | -------------------------------------------------------------------------------- /kubernetes/create.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | # Creates a K8 cluster on Hetzner. For configuration, look at fleet.yaml. 5 | # See: https://github.com/vitobotta/hetzner-k3s 6 | 7 | # Error out if dependencies aren't installed. 8 | if [[ ! -f "$(which hetzner-k3s)" ]] ; then 9 | LIGHT_RED='\033[1;31m' 10 | LIGHT_CYAN='\033[1;36m' 11 | echo -e "${LIGHT_RED}hetzner-k3s is not installed. Install it with the following commands:" 12 | echo -e "${LIGHT_CYAN} wget https://github.com/vitobotta/hetzner-k3s/releases/download/v1.0.2/hetzner-k3s-linux-x86_64" 13 | echo " chmod +x hetzner-k3s-linux-x86_64" 14 | echo " sudo mv hetzner-k3s-linux-x86_64 /usr/local/bin/hetzner-k3s" 15 | exit 1 16 | fi 17 | 18 | echo "Creating cluster..." 19 | 20 | # Load environment variables 21 | . .env 22 | export HETZNER_TOKEN="${HETZNER_TOKEN}" 23 | export K3S_VERSION="${K3S_VERSION}" 24 | export CLUSTER_NAME="${CLUSTER_NAME}" 25 | export CLUSTER_LOCATION="${CLUSTER_LOCATION}" 26 | export MASTERS_POOL_SIZE="${MASTERS_POOL_SIZE}" 27 | 28 | export WORKERS_POOL_SIZE="${WORKERS_POOL_SIZE_MIN}" 29 | export WORKERS_POOL_SIZE_MIN="${WORKERS_POOL_SIZE_MIN}" 30 | export WORKERS_POOL_SIZE_MAX="${WORKERS_POOL_SIZE_MAX}" 31 | export WORKERS_INSTANCE_TYPE="${WORKERS_INSTANCE_TYPE}" 32 | 33 | if (( "${WORKERS_POOL_SIZE_MIN}" == "${WORKERS_POOL_SIZE_MAX}" )); then 34 | export AUTOSCALING_ENABLED=false 35 | else 36 | export AUTOSCALING_ENABLED=true 37 | fi 38 | 39 | export LETSENCRYPT_EMAIL="${LETSENCRYPT_EMAIL}" 40 | hetzner-k3s create --config <(envsubst < ./fleet.yaml) 41 | 42 | # Copy kubeconfig to ~/.kube/config directory, backing up the old one if it exists 43 | export KUBECONFIG=./kubeconfig 44 | if [[ -f ~/.kube/config ]]; then 45 | BACKUP_CONFIG_PATH="$HOME/.kube/config.bak" 46 | echo "Copying old ~/.kube/config to ${BACKUP_CONFIG_PATH}" 47 | cp ~/.kube/config "${BACKUP_CONFIG_PATH}" 48 | fi 49 | cp ./kubeconfig ~/.kube/config 50 | 51 | # Once-off. If ran again, these are no-op 52 | helm repo add rancher-stable https://releases.rancher.com/server-charts/stable 53 | helm repo add jetstack https://charts.jetstack.io 54 | helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx 55 | # Update Helm chart repository cache 56 | helm repo update 57 | 58 | # Install the Ingress controller 59 | echo "Installing the ingress controller..." 60 | export RANCHER_HOSTNAME="${RANCHER_HOSTNAME}" 61 | helm upgrade --install ingress-nginx ingress-nginx/ingress-nginx \ 62 | -f <(envsubst < ./ingress-nginx-annotations.yaml) \ 63 | --namespace ingress-nginx \ 64 | --create-namespace 65 | kubectl apply -f ./ingress-nginx-configmap.yaml 66 | 67 | # Install metrics. Needed for HorizontalPodAutoscaler, etc. 68 | kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml 69 | 70 | # Install LetsEncrypt certificate manager 71 | # kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.10.1/cert-manager.crds.yaml 72 | echo "Installing LetsEncrypt certificate manager..." 73 | helm upgrade --install cert-manager jetstack/cert-manager \ 74 | --namespace cert-manager \ 75 | --create-namespace \ 76 | --version v1.10.1 \ 77 | --set installCRDs=true # TODO(acorn1010): Seems to be failing around here the first time? 78 | echo "Done installing LetsEncrypt. Applying lets-encrypt yaml..." 79 | envsubst < ./lets-encrypt.yaml | kubectl apply -f - 80 | 81 | # Wait for the Ingress controller to start 82 | # TODO(acorn1010): Improve this. Is it possible to use `kubectl -n ingress-nginx rollout status` here? 83 | echo "Waiting for Ingress Controller..." 84 | INGRESS_NGINX_STATUS=$(kubectl get pods --namespace=ingress-nginx | grep "1/"); 85 | while [ ! "${INGRESS_NGINX_STATUS}" ]; do 86 | echo "Waiting for Ingress Controller to finish starting up..."; 87 | sleep 2; 88 | INGRESS_NGINX_STATUS=$(kubectl get pods --namespace=ingress-nginx | grep "1/"); 89 | done 90 | 91 | # Include GCP secret so that we can deploy containers to GCR. 92 | export GCR_SECRET="${GCR_SECRET}" 93 | envsubst < ./gcp-secret.yaml | kubectl apply -f - 94 | 95 | # Now that an Ingress controller is installed, we can install Rancher 96 | echo "Installing Rancher..." 97 | RANCHER_PASS=$(mktemp -u XXXXXXXXXX) 98 | helm upgrade --install rancher rancher-stable/rancher \ 99 | --namespace cattle-system \ 100 | --create-namespace \ 101 | --set hostname="${RANCHER_HOSTNAME}" \ 102 | --set bootstrapPassword="${RANCHER_PASS}" \ 103 | --set ingress.ingressClassName=nginx \ 104 | --set ingress.tls.source=letsEncrypt \ 105 | --set letsEncrypt.email="${LETSENCRYPT_EMAIL}" \ 106 | --set letsEncrypt.ingress.class=nginx 107 | 108 | # Install services that we depend on 109 | ./services/redis.sh 110 | 111 | COLOR_OFF='\033[0m' 112 | CYAN_UNDERLINE='\033[4;36m' 113 | echo "Finished setting up K8." 114 | echo -e "You will need to visit ${CYAN_UNDERLINE}https://console.hetzner.cloud/projects/${COLOR_OFF} and find your Load Balancer's Public IP" 115 | echo "Set this Public IP as an A record in your DNS for ${RANCHER_HOSTNAME}" 116 | echo "" 117 | echo -e "Once your DNS has propagated, continue setting up Rancher here: ${CYAN_UNDERLINE}https://${RANCHER_HOSTNAME}/dashboard/?setup=${RANCHER_PASS}${COLOR_OFF}" 118 | -------------------------------------------------------------------------------- /kubernetes/fleet.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | hetzner_token: $HETZNER_TOKEN 3 | cluster_name: $CLUSTER_NAME 4 | kubeconfig_path: "./kubeconfig" 5 | k3s_version: $K3S_VERSION 6 | public_ssh_key_path: "~/.ssh/id_rsa.pub" 7 | private_ssh_key_path: "~/.ssh/id_rsa.pem" 8 | ssh_allowed_networks: 9 | - 0.0.0.0/0 10 | api_allowed_networks: 11 | - 0.0.0.0/0 12 | 13 | schedule_workloads_on_masters: false 14 | masters_pool: 15 | instance_type: cpx21 16 | instance_count: $MASTERS_POOL_SIZE 17 | # Valid locations: 18 | # Germany: nbg1, fsn1 19 | # Finland: hel1 20 | # US West: hil 21 | # US East: ash 22 | location: $CLUSTER_LOCATION 23 | labels: 24 | - key: purpose 25 | value: master 26 | # taints: 27 | # something: value1:NoSchedule 28 | worker_node_pools: 29 | - name: workers 30 | instance_type: $WORKERS_INSTANCE_TYPE 31 | instance_count: $WORKERS_POOL_SIZE 32 | autoscaling: 33 | enabled: $AUTOSCALING_ENABLED 34 | min_instances: $WORKERS_POOL_SIZE_MIN 35 | max_instances: $WORKERS_POOL_SIZE_MAX 36 | location: $CLUSTER_LOCATION 37 | labels: 38 | - key: purpose 39 | value: worker 40 | additional_packages: 41 | #- somepackage 42 | post_create_commands: 43 | - apt update 44 | - apt upgrade -y 45 | - apt autoremove -y 46 | - shutdown -r now 47 | enable_encryption: true 48 | -------------------------------------------------------------------------------- /kubernetes/gcp-secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: gcr-io 5 | namespace: default 6 | data: 7 | .dockerconfigjson: $GCR_SECRET 8 | type: kubernetes.io/dockerconfigjson 9 | -------------------------------------------------------------------------------- /kubernetes/ingress-nginx-annotations.yaml: -------------------------------------------------------------------------------- 1 | # INSTALLATION 2 | # 1. Install Helm: https://helm.sh/docs/intro/install/ 3 | # 2. Add ingress-nginx help repo: helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx 4 | # 3. Update information of available charts locally from chart repositories: helm repo update 5 | # 4. Install ingress-nginx: 6 | # helm upgrade --install \ 7 | # ingress-nginx ingress-nginx/ingress-nginx \ 8 | # -f ./ingress-nginx-annotations.yaml \ 9 | # --namespace ingress-nginx \ 10 | # --create-namespace 11 | 12 | # LIST of all ANNOTATIONS: https://github.com/hetznercloud/hcloud-cloud-controller-manager/blob/master/internal/annotation/load_balancer.go 13 | 14 | controller: 15 | kind: DaemonSet 16 | service: 17 | annotations: 18 | # Germany: 19 | # - nbg1 (Nuremberg) 20 | # - fsn1 (Falkensteing) 21 | # Finland: 22 | # - hel1 (Helsinki) 23 | # USA: 24 | # - ash (Ashburn, Virginia) 25 | # - hil (Hillsboro, Oregon) 26 | # Without this the load balancer won't be provisioned and will stay in "pending" state. 27 | # The state you can check via "kubectl get svc -n ingress-nginx" 28 | load-balancer.hetzner.cloud/location: $CLUSTER_LOCATION 29 | 30 | # Name of load balancer. This name you will see in your Hetzner's cloud console (site) at the "Your project -> Load Balancers" page 31 | load-balancer.hetzner.cloud/name: worker_LB 32 | 33 | # Ensures that the communication between the load balancer and the cluster nodes happens through the private network 34 | load-balancer.hetzner.cloud/use-private-ip: "true" 35 | 36 | # [ START: If you care about seeing the actual IP of the client then use these two annotations ] 37 | # - "uses-proxyprotocol" enables the proxy protocol on the load balancers so that ingress controller and 38 | # applications can "see" the real IP address of the client. 39 | # - "hostname" is needed just if you use cert-manager (LetsEncrypt SSL certificates). You need to use it in order 40 | # to fix fails http01 challenges of "cert-manager" (https://cert-manager.io/docs/). 41 | # Here (https://github.com/compumike/hairpin-proxy) you can find a description of this problem. 42 | # To be short: the easiest fix provided by some providers (including Hetzner) is to configure the load balancer so 43 | # that it uses a hostname instead of an IP. 44 | load-balancer.hetzner.cloud/uses-proxyprotocol: 'true' 45 | 46 | # 1. "yourDomain.com" must be configured in the DNS correctly to point to the Nginx load balancer, 47 | # otherwise the provision of certificates won't work; 48 | # 2. if you use a few domains, specify any one. 49 | load-balancer.hetzner.cloud/hostname: $RANCHER_HOSTNAME 50 | # [ END: If you care about seeing the actual IP of the client then use these two annotations ] 51 | 52 | load-balancer.hetzner.cloud/http-redirect-https: 'false' 53 | -------------------------------------------------------------------------------- /kubernetes/ingress-nginx-configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | # Do not change name - this is the name required by Nginx Ingress Controller 5 | name: ingress-nginx-controller 6 | namespace: ingress-nginx 7 | data: 8 | use-proxy-protocol: "true" 9 | -------------------------------------------------------------------------------- /kubernetes/lets-encrypt.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cert-manager.io/v1 2 | kind: ClusterIssuer 3 | metadata: 4 | # Note: ClusterIssuer isn't supposed to be in a namespace 5 | name: letsencrypt-prod 6 | spec: 7 | acme: 8 | # You must replace this email address with your own. 9 | # Let's Encrypt will use this to contact you about expiring 10 | # certificates, and issues related to your account. 11 | email: $LETSENCRYPT_EMAIL 12 | server: https://acme-v02.api.letsencrypt.org/directory 13 | privateKeySecretRef: 14 | name: letsencrypt-prod-account-key 15 | # Add a single challenge solver, HTTP01 using nginx 16 | solvers: 17 | - http01: 18 | ingress: 19 | class: nginx 20 | -------------------------------------------------------------------------------- /kubernetes/services/README.md: -------------------------------------------------------------------------------- 1 | Extra services that not every cloud needs. These provide things like: 2 | - Cache: Redis 3 | - Persistent Storage (Rook/Ceph or Minio). Minio looks pretty, but may not allow commercial use for free? 4 | - Pub/Sub listeners from database: Kafka 5 | - Persistent Database: Vitess 6 | 7 | -------------------------------------------------------------------------------- /kubernetes/services/redis.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Handles application of the Redis Cluster kubectl. Required because ConfigMap needs a password. 3 | set -e 4 | 5 | . ../.env 6 | export REDIS_PASSWORD="${REDIS_PASSWORD}" 7 | envsubst < ./redis.yaml | kubectl --kubeconfig ~/.kube/config.hetzner apply -f - 8 | 9 | -------------------------------------------------------------------------------- /kubernetes/services/redis.yaml: -------------------------------------------------------------------------------- 1 | # TODO(acorn1010): Switch with a nice clustered Redis with lots of persistence-y stuff. 2 | --- 3 | apiVersion: v1 4 | kind: Namespace 5 | metadata: 6 | name: redis 7 | --- 8 | apiVersion: v1 9 | kind: ConfigMap 10 | metadata: 11 | name: redis-configmap 12 | namespace: redis 13 | data: 14 | redis-stack.conf: | 15 | requirepass $REDIS_PASSWORD 16 | dir /mnt/data 17 | appendonly yes 18 | daemonize yes 19 | --- 20 | apiVersion: v1 21 | kind: PersistentVolumeClaim 22 | metadata: 23 | name: pvc-redis 24 | namespace: redis 25 | spec: 26 | accessModes: 27 | - ReadWriteOnce 28 | resources: 29 | requests: 30 | storage: 40Gi 31 | --- 32 | apiVersion: apps/v1 33 | kind: StatefulSet 34 | metadata: 35 | name: redis-statefulset 36 | namespace: redis 37 | spec: 38 | serviceName: redis-service 39 | replicas: 1 40 | selector: 41 | matchLabels: 42 | app: redis 43 | template: 44 | metadata: 45 | labels: 46 | app: redis 47 | spec: 48 | containers: 49 | - name: redis-container 50 | image: redis/redis-stack-server:6.2.6-v7 51 | env: 52 | - name: REDIS_DATA_DIR # Required. Seems to overwrite the ConfigMap 53 | value: "/mnt/data" 54 | ports: 55 | - containerPort: 6379 56 | volumeMounts: 57 | - name: mnt-data 58 | mountPath: /mnt/data 59 | - name: config 60 | mountPath: /redis-stack.conf 61 | subPath: ./redis-stack.conf 62 | readOnly: true 63 | volumes: 64 | - name: mnt-data 65 | persistentVolumeClaim: 66 | claimName: pvc-redis 67 | - name: config 68 | configMap: 69 | name: redis-configmap 70 | --- 71 | apiVersion: v1 72 | kind: Service 73 | metadata: 74 | name: redis-service 75 | namespace: redis 76 | spec: 77 | selector: 78 | app: redis 79 | ports: 80 | - name: redis 81 | port: 6379 82 | # TODO(acorn1010): When we get 3+ nodes, switch to Redis Enterprise: 83 | # https://docs.redis.com/latest/kubernetes/deployment/quick-start/ 84 | -------------------------------------------------------------------------------- /kubernetes/storage.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: hcloud-csi 5 | namespace: kube-system 6 | stringData: 7 | token: $HETZNER_TOKEN 8 | -------------------------------------------------------------------------------- /kubernetes/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Namespace 4 | metadata: 5 | name: $NAMESPACE 6 | --- 7 | apiVersion: v1 8 | kind: Secret 9 | metadata: 10 | name: gcr-io 11 | namespace: $NAMESPACE 12 | data: 13 | .dockerconfigjson: $GCR_SECRET 14 | type: kubernetes.io/dockerconfigjson 15 | --- 16 | apiVersion: apps/v1 17 | kind: Deployment 18 | metadata: 19 | name: ${NAMESPACE}-deployment 20 | namespace: $NAMESPACE 21 | spec: 22 | replicas: $REPLICAS # Number of servers. Increase as desired 23 | selector: 24 | matchLabels: 25 | app: ${NAMESPACE}-server 26 | template: 27 | metadata: 28 | labels: 29 | app: ${NAMESPACE}-server 30 | spec: 31 | imagePullSecrets: 32 | - name: gcr-io 33 | containers: 34 | - name: ${NAMESPACE}-server 35 | image: $DOCKER_IMAGE 36 | imagePullPolicy: IfNotPresent 37 | # Limit game server resources. If this isn't enough for 1,000 players then we can 38 | # increase it later. 39 | resources: 40 | requests: 41 | cpu: $CPU # Request 0.5 vCPUs. 42 | memory: $MEMORY # Request 1 GiB of RAM 43 | ports: 44 | - containerPort: 3000 45 | --- 46 | apiVersion: v1 47 | kind: Service 48 | metadata: 49 | name: ${NAMESPACE}-service 50 | namespace: $NAMESPACE 51 | spec: 52 | ports: 53 | - name: http 54 | protocol: TCP 55 | port: 3000 56 | targetPort: 3000 57 | selector: 58 | app: ${NAMESPACE}-server 59 | --- 60 | apiVersion: networking.k8s.io/v1 61 | kind: Ingress 62 | metadata: 63 | name: ${NAMESPACE}-ingress 64 | namespace: $NAMESPACE 65 | annotations: 66 | kubernetes.io/tls-acme: "true" 67 | nginx.ingress.kubernetes.io/rewrite-target: / 68 | cert-manager.io/cluster-issuer: "letsencrypt-prod" 69 | spec: 70 | ingressClassName: nginx 71 | rules: 72 | - host: $HOST 73 | http: 74 | paths: 75 | - path: / 76 | pathType: Prefix 77 | backend: 78 | service: 79 | name: ${NAMESPACE}-service 80 | port: 81 | number: 3000 82 | tls: 83 | - hosts: 84 | - $HOST 85 | secretName: ${HOST}-tls 86 | -------------------------------------------------------------------------------- /scripts/perf_test.mjs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env zx 2 | 3 | /** 4 | * A FILO (First-In Last-Out) fetcher that limits the number of concurrent 5 | * fetches. 6 | */ 7 | class BatchedFetcher { 8 | /** 9 | * The maximum number of outstanding requests that can be simultaneously 10 | * processed. 11 | */ 12 | maxRequests; 13 | 14 | /** Set of outstanding request promises. */ 15 | outstandingRequests = new Set(); 16 | 17 | constructor({maxRequests}) { 18 | if (maxRequests < 1) { 19 | throw new Error(`Invalid maxRequests (must be >= 1): ${maxRequests}`); 20 | } 21 | this.maxRequests = maxRequests; 22 | } 23 | 24 | /** 25 | * Returns the result of fetching `url`. Won't return until the fetcher has 26 | * had time to process this request, which could be a while if there are more 27 | * `outstandingRequests` than `maxRequests`. 28 | */ 29 | async fetch(url) { 30 | // TODO(acorn1010): May have performance issues when outstandingRequests 31 | // greatly exceeds maxRequests. 32 | while (this.outstandingRequests.size >= this.maxRequests) { 33 | await Promise.any(this.outstandingRequests); 34 | } 35 | const promise = fetch(url); 36 | this.outstandingRequests.add(promise); 37 | promise.finally(() => { 38 | this.outstandingRequests.delete(promise); 39 | }); 40 | return promise; 41 | } 42 | } 43 | 44 | function generateRandomString() { 45 | const alphabet = 'abcdefghijklmnopqrstuvwxyz0123456789'; 46 | return Array.from({length: 5}, () => alphabet[Math.floor(Math.random() * alphabet.length)]).join(''); 47 | } 48 | 49 | const fetcher = new BatchedFetcher({maxRequests: 30}); 50 | const start = Date.now(); 51 | const promises = []; 52 | const cacheBuster = generateRandomString(); 53 | for (let i = 0; i < 1_000; ++i) { 54 | const url = `https://render.acorn1010.com/https://foony.com/404_${cacheBuster}_${i}`; 55 | promises.push(fetcher.fetch(url).then(async (response) => { 56 | const size = (await response.blob()).size; 57 | console.log(url, size); 58 | })); 59 | } 60 | 61 | const result = await Promise.all(promises); 62 | const ms = Date.now() - start; 63 | console.log(`Took ${ms} ms, or ${ms / promises.length} ms average`); 64 | 65 | console.log(result); 66 | -------------------------------------------------------------------------------- /server/.env.example: -------------------------------------------------------------------------------- 1 | 2 | # Add your unique password for creating sessions 3 | SESSION_PASSWORD=hunter2 4 | 5 | # Add your Redis Sentinel password here. Needed if you're using the Redis service. 6 | REDIS_PASSWORD=hunter2 7 | -------------------------------------------------------------------------------- /server/Dockerfile: -------------------------------------------------------------------------------- 1 | # NOTE: It'd be nice to use Alpine, but Chrome is a chonky boi with lots of deps, so we're going with a chonky image 2 | FROM mcr.microsoft.com/playwright:v1.30.0-focal 3 | 4 | ENV PROD 1 5 | 6 | WORKDIR /usr/src/app 7 | 8 | # Configure the base npm application 9 | # Instead of a simple command like COPY ./server ./server/, we split this into multiple commands in 10 | # order to ignore the node_modules library (for some reason .dockerignore is being ignored). 11 | COPY ./server/.env ./server/tsconfig.json ./server/package.json ./server/package-lock.json ./server/ 12 | COPY ./server/lib ./server/lib 13 | 14 | COPY ./shared ./shared/ 15 | 16 | # TODO(acorn1010): Figure out why COPY isn't copying as pptruser. 17 | 18 | WORKDIR /usr/src/app/server 19 | RUN npm ci 20 | 21 | #CMD ["sleep", "6000000"] 22 | ENTRYPOINT ["npm", "start"] 23 | 24 | # Expose Ports. 25 | EXPOSE 3000 26 | -------------------------------------------------------------------------------- /server/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | # This is the only section you should modify if creating a new deployment. 5 | # TODO(acorn1010): Move the rest of this script into the kubernetes/ folder 6 | export HOST="api.rendermy.site" 7 | export NAMESPACE="prerender" 8 | export REPLICAS=3 # Number of server instances to spin up 9 | export CPU="700m" # Request 0.7 vCPUs 10 | export MEMORY="1.7Gi" # Request 1.7 GiB of RAM 11 | 12 | # EVERYTHING BELOW THIS LINE SHOULD STAY THE SAME ACROSS ALL DEPLOYMENTS. 13 | echo "Deploying server." 14 | 15 | # Source environment variables (needed for $PROJECT_ID) 16 | . ../kubernetes/.env 17 | 18 | # Path to where the container will be uploaded. We use gcr.io (Google Container Registry) 19 | DOCKER_IMAGE=gcr.io/${PROJECT_ID}/prerender 20 | # Container version (e.g. "2021.02.03_11.15.30") 21 | VERSION=$(date +"%Y.%m.%d_%H.%M.%S") 22 | 23 | # Compile code. 24 | (cd src; npm install && npm run build) 25 | 26 | # Build Docker image. We navigate up to our parent directory so that we can include the shared/ library. 27 | (cd ..; docker build --platform linux/amd64 -t ${DOCKER_IMAGE}:latest -t ${DOCKER_IMAGE}:"${VERSION}" --progress=plain -f server/Dockerfile .) 28 | 29 | # Deploy to container registry. If this fails, you may need to run the below command: 30 | # gcloud auth configure-docker 31 | docker push ${DOCKER_IMAGE}:"${VERSION}" 32 | docker push ${DOCKER_IMAGE}:latest 33 | 34 | # Replace environment variables in fleet.yaml and update our fleet with the latest image. 35 | export DOCKER_IMAGE="${DOCKER_IMAGE}:${VERSION}" 36 | export GCR_SECRET="${GCR_SECRET}" # Needed so that K3s can pull the docker image 37 | 38 | kubeConfig=~/.kube/config.joes 39 | envsubst < ../kubernetes/templates/deployment.yaml | kubectl --kubeconfig "${kubeConfig}" apply -f - 40 | -------------------------------------------------------------------------------- /server/forward-redis.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | echo "Starting port forwarding. You'll need your REDIS_PASSWORD from ../kubernetes/.env to connect." 5 | echo "Run this command in a safe terminal to get it:" 6 | echo "cat ../kubernetes/.env | grep REDIS_PASSWORD" 7 | kubectl port-forward --kubeconfig=../kubernetes/kubeconfig --namespace=redis redis-statefulset-0 6379:6379 8 | 9 | -------------------------------------------------------------------------------- /server/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "prerender-server", 3 | "version": "1.0.0", 4 | "description": "Service for rendering webpages with headless Chrome.", 5 | "author": "Acorn1010", 6 | "main": "lib/server/src/index.js", 7 | "types": "lib/server/src/index.d.ts", 8 | "files": [ 9 | "lib/*" 10 | ], 11 | "scripts": { 12 | "__comment": "tsc doesn't respect non-ts files, so we have to roll our own file copy in the build process: https://github.com/microsoft/TypeScript/issues/30835. TODO(acorn1010): try out tsc -w for start.", 13 | "lint": "eslint 'src/**'", 14 | "prebuild": "del lib/", 15 | "build": "(cd ../shared && npm run build) && tsc && tsc-alias", 16 | "postbuild": "cpy 'src/**/*' '!**/*.ts' '!**/*.d.ts' '!**/*.d.ts.map' lib/server/src", 17 | "start": "node lib/server/src/index.js" 18 | }, 19 | "engines": { 20 | "node": "18" 21 | }, 22 | "dependencies": { 23 | "@fastify/cors": "^8.2.1", 24 | "@types/compression": "^1.7.2", 25 | "@types/cors": "^2.8.13", 26 | "ajv": "^8.12.0", 27 | "compression": "^1.7.4", 28 | "dotenv": "^16.0.3", 29 | "fastify": "^4.15.0", 30 | "firebase-admin": "^11.5.0", 31 | "ioredis": "^5.3.0", 32 | "lodash": "^4.17.21", 33 | "lru-cache": "^7.16.1", 34 | "minimatch": "^7.4.3", 35 | "nanoid": "^3.3.4", 36 | "node-cache": "^5.1.2", 37 | "node-fetch": "^3.3.1", 38 | "playwright": "^1.32.2", 39 | "url": "^0.11.0", 40 | "zod": "^3.21.0" 41 | }, 42 | "devDependencies": { 43 | "@types/lodash": "^4.14.191", 44 | "@types/node": "^18.11.18", 45 | "cpy-cli": "^4.2.0", 46 | "del-cli": "^5.0.0", 47 | "tsc-alias": "^1.8.5", 48 | "typescript": "^5.0.4" 49 | }, 50 | "private": true 51 | } 52 | -------------------------------------------------------------------------------- /server/src/Environment.ts: -------------------------------------------------------------------------------- 1 | import Redis from "ioredis"; 2 | import * as dotenv from 'dotenv'; 3 | import {RedisWrapper} from "./db/RedisWrapper"; 4 | import Ajv from "ajv"; 5 | import {Actions} from '@shared/Action'; 6 | import * as apiSchema from '@shared/api_schema.json'; 7 | import {ServerActions} from "./api"; 8 | dotenv.config(); 9 | 10 | type ValidateFunction = import('ajv').ValidateFunction; 11 | 12 | /** The global environment. */ 13 | export const env = { 14 | redis: new RedisWrapper( 15 | new Redis({ 16 | host: 'redis-service.redis.svc.cluster.local', 17 | /** Use DB 1 because we're using db-0 for Foony. */ 18 | db: 1, 19 | password: process.env.REDIS_PASSWORD, 20 | }) 21 | ), 22 | validators: makeValidators(), 23 | // Vitess <-- maybe just Redis? PostgresQL is another good choice here 24 | // Kafka <-- maybe just Redis for now? 25 | } as const; 26 | 27 | /** Create all API schema validators. Not all Action Options have a validator. */ 28 | function makeValidators(): Map { 29 | const ajv = (new Ajv()).addSchema(apiSchema); 30 | const result = new Map(); 31 | 32 | // Note: It's possible below that the schemas are malformed. We use ! operators anyways because if 33 | // a schema is missing, it's unsafe to run the game server, so it's better to fail. 34 | 35 | // Next, we add all of the non-generic actions. 36 | // const actionOptions = ajv.getSchema('#/definitions/Actions')!; 37 | // const properties = (actionOptions.schema as SchemaObject)?.properties!; 38 | for (const action in ServerActions) { 39 | const maybeValidator = ajv.getSchema(`#/definitions/ActionInputs/properties/${action}`); 40 | if (!maybeValidator) { 41 | throw new Error(`Missing API action schema: ${action}`); 42 | } 43 | result.set(action as keyof typeof ServerActions, maybeValidator); 44 | } 45 | 46 | return result; 47 | } 48 | -------------------------------------------------------------------------------- /server/src/ServerAction.ts: -------------------------------------------------------------------------------- 1 | import {Actions} from "@shared/Action"; 2 | import {CallableContext} from "./http/CallableContext"; 3 | 4 | type Promisable = T | Promise; 5 | 6 | export type ServerAction = 7 | ((context: CallableContext, ...args: Actions[K]['input']) => Promisable); 8 | -------------------------------------------------------------------------------- /server/src/TimeUtils.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Returns the current 'yyyy-mm'. Used in Redis for bucketing by month. If `monthOffset` is defined, 3 | * this will be the number of months in the past / future (positive for future, negative for past). 4 | */ 5 | export function getYyyyMm(monthOffset = 0) { 6 | const now = new Date(); 7 | now.setMonth(now.getMonth() + monthOffset); 8 | let year = now.getFullYear(); 9 | let month = now.getMonth() + 1; 10 | return `${year}-${month < 10 ? '0' : ''}${month}`; 11 | } 12 | 13 | /** Returns the current 'yyyy-mm-dd'. Used in Redis for bucketing by month. */ 14 | export function getYyyyMmDd() { 15 | const now = new Date(); 16 | const year = now.getFullYear(); 17 | const month = now.getMonth() + 1; 18 | const day = now.getDate(); 19 | return `${year}-${month < 10 ? '0' : ''}${month}-${day < 10 ? '0' : ''}${day}`; 20 | } 21 | -------------------------------------------------------------------------------- /server/src/api/flush.ts: -------------------------------------------------------------------------------- 1 | import {env} from "../Environment"; 2 | import {CallableContext} from "../http/CallableContext"; 3 | import {getUserId} from "../http/AuthUtils"; 4 | import {ServerAction} from "../ServerAction"; 5 | 6 | export const flush: ServerAction<'flush'> = async (context: CallableContext) => { 7 | const userId = getUserId(context); 8 | 9 | // TODO(acorn1010): Allow partial cache invalidations (e.g. "com.*", "com.foony.*"). 10 | const result = await env.redis.url.flush(userId); 11 | console.log('Deleted all cache for user!', userId, result); 12 | return true; 13 | }; 14 | -------------------------------------------------------------------------------- /server/src/api/getMonthlyRenderCounts.ts: -------------------------------------------------------------------------------- 1 | import {env} from "../Environment"; 2 | import {CallableContext} from "../http/CallableContext"; 3 | import {getUserId} from "../http/AuthUtils"; 4 | import {ServerAction} from "../ServerAction"; 5 | 6 | export const getMonthlyRenderCounts: ServerAction<'getMonthlyRenderCounts'> = async (context: CallableContext) => { 7 | const userId = getUserId(context); 8 | 9 | // TODO(acorn1010): Allow partial cache invalidations (e.g. "com.*", "com.foony.*"). 10 | return env.redis.user.getMonthlyRenderCounts(userId); 11 | }; 12 | -------------------------------------------------------------------------------- /server/src/api/getProfile.ts: -------------------------------------------------------------------------------- 1 | import {env} from "../Environment"; 2 | import {CallableContext} from "../http/CallableContext"; 3 | import {getUserId} from "../http/AuthUtils"; 4 | import {ServerAction} from "../ServerAction"; 5 | 6 | export const getProfile: ServerAction<'getProfile'> = async (context: CallableContext) => { 7 | return env.redis.user.get(getUserId(context)); 8 | }; 9 | -------------------------------------------------------------------------------- /server/src/api/index.ts: -------------------------------------------------------------------------------- 1 | import {Actions} from "@shared/Action"; 2 | import {ServerAction} from "../ServerAction"; 3 | import {flush} from "./flush"; 4 | import {getMonthlyRenderCounts} from "./getMonthlyRenderCounts"; 5 | import {getProfile} from './getProfile'; 6 | import {refreshToken} from "./refreshToken"; 7 | 8 | /** 9 | * All API server actions. When you add an action in shared/Action.ts, you'll need to add it here 10 | * too. The client calls these methods by using `call.flush()` 11 | */ 12 | export const ServerActions = { 13 | flush, 14 | getMonthlyRenderCounts, 15 | getProfile, 16 | refreshToken, 17 | } satisfies {[K in keyof Actions]: ServerAction}; 18 | -------------------------------------------------------------------------------- /server/src/api/refreshToken.ts: -------------------------------------------------------------------------------- 1 | import {env} from "../Environment"; 2 | import {CallableContext} from "../http/CallableContext"; 3 | import {getUserId} from "../http/AuthUtils"; 4 | import {ServerAction} from "../ServerAction"; 5 | 6 | export const refreshToken: ServerAction<'refreshToken'> = async (context: CallableContext) => { 7 | return env.redis.user.refreshToken(getUserId(context)); 8 | }; 9 | -------------------------------------------------------------------------------- /server/src/api/render.ts: -------------------------------------------------------------------------------- 1 | import {RenderResponse} from "../db/models/UrlModel"; 2 | import {env} from "../Environment"; 3 | import {render} from "../browsers/ChromeBrowser"; 4 | import {FastifyReply, FastifyRequest} from "fastify"; 5 | import {HttpsError} from "../http/HttpsError"; 6 | import {getUserId} from "../http/AuthUtils"; 7 | import {CallableContext} from "../http/CallableContext"; 8 | 9 | // Don't include some problematic headers from the original third-party response. We're using our 10 | // own content-encoding, we don't support keep-alive connections, and we don't do chunked encoding. 11 | const HEADER_BLACK_LIST = new Set(['transfer-encoding', 'connection', 'content-encoding']); 12 | 13 | export async function doRequest(req: FastifyRequest, res: FastifyReply, context: CallableContext) { 14 | let url = req.url.slice('/'.length); 15 | if (!url || !url.match(/^https?:\/\//)) { 16 | throw new HttpsError('failed-precondition', `Invalid URL "${req.url}". Example request: https://api.rendermy.site/https://foony.com`); 17 | } 18 | 19 | // If this is not a full URL, then base the URL off of where it's requested from. This isn't 20 | // really necessary and could be deleted without affecting the service. It's more for local 21 | // development. 22 | if (url.indexOf('://') < 0) { 23 | const referer = req.headers.referer || ''; 24 | // From https://www.rfc-editor.org/rfc/rfc3986#page-51. 25 | // Given a referer of e.g. http://localhost:3000/https://example.com/foo/bar, 26 | // returns http://localhost:3000/https://example.com 27 | const baseReferer = referer.match(/((?:([^:\/?#]+):)?\/\/([^\/?#]*)\/(?:([^:\/?#]+):)?\/\/([^\/?#]*)).*/)?.[1]; 28 | url = (baseReferer ?? referer) + '/' + url; 29 | } 30 | console.log('Navigating to URL', url); 31 | 32 | const userId = getUserId(context); 33 | let response: RenderResponse | null = null; 34 | const cachedResult = await env.redis.url.queryPage({ 35 | url, userAgent: req.headers['user-agent'], userId 36 | }); 37 | if (cachedResult) { 38 | response = cachedResult; 39 | // TODO(acorn1010): Remove after 2023-05-01 40 | response.headers ||= (response as any).responseHeaders; 41 | response.console ||= []; 42 | } 43 | 44 | // TODO(acorn1010): Instead of trying to render this directly, stick it in a worker queue 45 | // and wait for it to be finished. This will reduce API flakiness. 46 | if (!response) { 47 | console.log(`No cache. Rendering URL: ${url}`); 48 | response = await renderAndCache(userId, url, getRequestHeaders(req)); 49 | } 50 | 51 | if (!response) { 52 | throw new HttpsError('failed-precondition', `Invalid URL. Got: "${url}". Example request: "https://api.rendermy.site/https://foony.com".`); 53 | } 54 | 55 | // For each header in the actual response, set them 56 | for (const [key, value] of Object.entries(response.headers)) { 57 | // this would have been easier to write in SQL @matty_twoshoes 58 | if (!HEADER_BLACK_LIST.has(key.toLowerCase())) { 59 | res.header(key, value.indexOf('\n') >= 0 ? value.split('\n') : value); 60 | } 61 | } 62 | res.send(response.buffer); 63 | } 64 | 65 | function getRequestHeaders(req: FastifyRequest): Record { 66 | return Object.fromEntries( 67 | Object.entries(req.headers) 68 | .filter(([key, value]) => { 69 | if (typeof value !== 'string') { 70 | console.log('FILTERING', {key, value}); 71 | } 72 | // TODO(acorn1010): Convert string[] to .join('\n') 73 | return typeof value === 'string'; 74 | })) as Record; 75 | } 76 | 77 | /** Renders a `URL` on behalf of `userId` and caches the result if successful. */ 78 | export async function renderAndCache( 79 | userId: string, 80 | url: string, 81 | headers: Record): Promise { 82 | const result = await render(url, headers).catch(e => { 83 | console.error('Unable to render URL.', e); 84 | return null; 85 | }); 86 | if (!result) { 87 | return null; 88 | } 89 | // TODO(acorn1010): Should we return early instead of saving if it failed to render? 90 | // (e.g. 300+ error) 91 | env.redis.url.setPage({userId, url, renderResponse: result}).catch(e => { 92 | console.error('Failed to set page.', e); 93 | }); 94 | return result; 95 | } 96 | -------------------------------------------------------------------------------- /server/src/browsers/BrowserUtils.ts: -------------------------------------------------------------------------------- 1 | import {RenderResponse} from "../db/models/UrlModel"; 2 | import {ConsoleMessage, JSHandle, Page} from "playwright"; 3 | 4 | export type ConsoleMessageType = 5 | 'log' | 'debug' | 'info' | 'error' | 'warning' | 'dir' | 'dirxml' | 'table' | 'trace' | 'clear' 6 | | 'startGroup' | 'startGroupCollapsed' | 'endGroup' | 'assert' | 'profile' | 'profileEnd' 7 | | 'count' | 'timeEnd'; 8 | 9 | /** 10 | * Waits for the DOM to finish rendering. If there are no DOM changes for `debounceMs`, then the 11 | * page is considered to be done rendering. 12 | */ 13 | export async function waitForDomToSettle(page: Page, timeoutMs = 5_000, debounceMs = 750): Promise { 14 | const url = page.url(); 15 | return page.evaluate( 16 | ([timeoutMs, debounceMs, url]) => { 17 | function debounce(func: T, ms = 1_000): T { 18 | let timeout: ReturnType; 19 | return ((...args: unknown[]) => { 20 | clearTimeout(timeout); 21 | // @ts-ignore 22 | timeout = setTimeout(() => func.apply(this, args), ms); 23 | }) as any; 24 | } 25 | 26 | // TODO(acorn1010): Try to close JavaScript dialogs after 1 second 27 | console.log('Beginning to inject!'); 28 | window.addEventListener('error', function(event) { 29 | console.log('error was logged', event); 30 | event.stopImmediatePropagation(); 31 | }, true); 32 | 33 | return new Promise((resolve, reject) => { 34 | const mainTimeout = setTimeout(() => { 35 | observer.disconnect(); 36 | reject(new Error("Timed out while waiting for DOM to settle: " + url)); 37 | }, timeoutMs); 38 | 39 | const debouncedResolve = debounce(() => { 40 | observer.disconnect(); 41 | clearTimeout(mainTimeout); 42 | resolve(); 43 | }, debounceMs); 44 | 45 | const observer = new MutationObserver(() => debouncedResolve()); 46 | const config = {attributes: true, childList: true, subtree: true}; 47 | observer.observe(document.body, config); 48 | // It's possible this was a static page, which won't change the DOM, so fire off an 49 | // initial debounce. 50 | debouncedResolve(); 51 | }); 52 | }, 53 | [timeoutMs, debounceMs, url] as const, 54 | ); 55 | } 56 | 57 | /** Prints the `page`'s console to stdout. Useful in debugging. */ 58 | export function logConsole(page: Page, consoleOutput: {type: ConsoleMessageType, args: string[]}[]) { 59 | const describe = (jsHandle: JSHandle) => { 60 | return jsHandle.evaluate((obj: any) => { 61 | function safeStringify(value: any) { 62 | const seen = new WeakSet(); 63 | const replacer = (key: any, val: any) => { 64 | if (typeof val === "object" && val !== null) { 65 | if (seen.has(val)) { 66 | return "(circular ref)"; 67 | } 68 | seen.add(val); 69 | } 70 | return val; 71 | }; 72 | return JSON.stringify(value, replacer); 73 | } 74 | 75 | try { 76 | return safeStringify(obj); 77 | } catch (e: any) { 78 | return `Unable to safeStringify: ${typeof obj} (${e.message})`; 79 | } 80 | }, jsHandle); 81 | } 82 | 83 | // listen to browser console there 84 | page.on('console', async (message: ConsoleMessage) => { 85 | const args = await Promise.all(message.args().map(arg => describe(arg))); 86 | consoleOutput.push({type: message.type() as ConsoleMessageType, args}); 87 | }); 88 | } 89 | 90 | /** Fetches a static page. Doesn't do any rendering. Used as a fallback in case the render fails. */ 91 | export async function fetchPage(url: string, requestHeaders: Record): Promise> { 92 | try { 93 | const response = await fetch(url/*, {headers: requestHeaders}*/); 94 | // FIXME(acorn1010): Something in here is failing. What could it be? 95 | // const response = await fetch(url, { 96 | // "host": "localhost:3000", 97 | // "connection": "keep-alive", 98 | // "upgrade-insecure-requests": "1", 99 | // "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/110.0.5478.0 Safari/537.36", 100 | // "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", 101 | // "sec-fetch-site": "none", 102 | // "sec-fetch-mode": "navigate", 103 | // "sec-fetch-user": "?1", 104 | // "sec-fetch-dest": "document", 105 | // "accept-encoding": "gzip, deflate, br" 106 | // }); 107 | return { 108 | statusCode: response.status, 109 | console: [], 110 | headers: Object.fromEntries(response.headers.entries()), 111 | buffer: Buffer.from(await response.arrayBuffer()), 112 | }; 113 | } catch (e) { 114 | console.error(`render URL fetch failed: ${url}`, e); 115 | } 116 | return {statusCode: 500, console: [], headers: {}, buffer: Buffer.from([])}; 117 | } 118 | -------------------------------------------------------------------------------- /server/src/browsers/ChromeBrowser.ts: -------------------------------------------------------------------------------- 1 | import playwright, {Browser, BrowserContext} from "playwright"; 2 | import {fetchPage, logConsole, waitForDomToSettle} from "./BrowserUtils"; 3 | import {env} from "../Environment"; 4 | import {shuffle} from "lodash"; 5 | import {renderAndCache} from "../api/render"; 6 | import {RenderResponse} from "../db/models/UrlModel"; 7 | import minimatch from "minimatch"; 8 | import { User } from "@shared/models/User"; 9 | 10 | /** Maximum lifetime of the browser before it gets killed and recreated. */ 11 | const BROWSER_MAX_LIFETIME_MS = 10 * 60_000; 12 | 13 | const MAX_OUTSTANDING_REQUESTS = 10; 14 | 15 | const PAGE_EXPIRATION_MS = 30_000; 16 | 17 | class ChromeBrowserRunnerSingleton { 18 | private browser: Promise | null = null; 19 | private userContexts = new Map(); 20 | 21 | /** Timestamp when the browser was last created. Used to refresh the browser after a while. */ 22 | private createdAt = 0; 23 | 24 | /** All outstanding requests. These need to be resolved before the browser can be closed. */ 25 | private outstandingRequests = new Set>(); 26 | 27 | /** Returns the current number of outstanding requests. */ 28 | getOutstandingRequestsCount() { 29 | return this.outstandingRequests.size; 30 | } 31 | 32 | /** (Re)-initializes the browser. */ 33 | private async init() { 34 | if (!this.browser || (this.createdAt + BROWSER_MAX_LIFETIME_MS < Date.now())) { 35 | await this.recreateBrowser(); 36 | } 37 | return this.browser!; 38 | } 39 | 40 | private async recreateBrowser() { 41 | console.log('Recreating browser...'); 42 | this.closeBrowser().then(() => {}); 43 | this.browser = playwright.chromium.launch({ 44 | args: ['--hide-scrollbars', '--disable-gpu'], 45 | }); 46 | this.createdAt = Date.now(); 47 | } 48 | 49 | /** Closes the browser gracefully after all outstanding requests are complete. */ 50 | private async closeBrowser() { 51 | if (!this.browser) { 52 | return; 53 | } 54 | 55 | const start = Date.now(); 56 | // Copy outstanding requests. New requests may come in while this is processing. 57 | const requests = [...this.outstandingRequests]; 58 | const browserPromise = this.browser; 59 | const browser = await browserPromise; 60 | await Promise.all(requests); // Wait for requests to complete 61 | 62 | // Close all outstanding contexts 63 | const contextPromises = []; 64 | for (const context of this.userContexts.values()) { 65 | contextPromises.push(context.close().catch(e => console.error('Error closing context: ', e))); 66 | } 67 | this.userContexts.clear(); 68 | await Promise.all(contextPromises); 69 | 70 | // Close gracefully 71 | console.log('Closing browser...'); 72 | await browser.close(); 73 | console.log('Browser closed!'); 74 | // If this browser is still the same one as the promise, then it doesn't exist anymore, so set 75 | // it to null. 76 | if (this.browser === browserPromise) { 77 | this.browser = null; 78 | this.userContexts.clear(); 79 | } 80 | console.log('Closing the browser took', (Date.now() - start) / 1000, 'seconds'); 81 | } 82 | 83 | public async render(url: string, requestHeaders: Record): Promise { 84 | const start = Date.now(); 85 | // While we're at our limit of outstanding requests, wait for one to complete. 86 | while (this.outstandingRequests.size >= MAX_OUTSTANDING_REQUESTS) { 87 | await Promise.any(this.outstandingRequests); 88 | } 89 | 90 | // TODO(acorn1010): Browser can crash if it fails to render here. Do we care? Can happen because 91 | // of 20s timeout. 92 | const response = this.renderPage(url, requestHeaders); 93 | console.log(`Adding outstanding request for URL: ${url}`); 94 | this.outstandingRequests.add(response); 95 | response.finally(() => { 96 | console.log(`Removing outstanding request for URL: ${url}`); 97 | // Remove the outstanding response when it's completed 98 | this.outstandingRequests.delete(response); 99 | }); 100 | const result = await response; 101 | return {...result, renderTimeMs: Date.now() - start}; 102 | } 103 | 104 | private async renderPage(url: string, requestHeaders: Record): Promise> { 105 | console.log(`Rendering page: ${url}`); 106 | const browser = await this.init(); 107 | // Reuse contexts by the same user / group of users 108 | const contextKey = getContextKey(requestHeaders); 109 | const context = this.userContexts.get(contextKey) || await browser.newContext(); 110 | if (context) { 111 | this.userContexts.set(contextKey, context); 112 | } 113 | const page = await context.newPage(); 114 | try { 115 | // TODO(acorn1010): Include requestHeaders, but exclude User-Agent? 116 | // await page.setExtraHTTPHeaders({ 117 | // // ...requestHeaders, 118 | // "user-agent": `Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/110.0.5478.${Math.floor(Math.random() * 100_000)} Safari/537.36`, 119 | // }); 120 | page.setDefaultTimeout(PAGE_EXPIRATION_MS); // TODO(acorn1010): Is this still necessary? 121 | 122 | const responseConsole: RenderResponse['console'] = []; 123 | logConsole(page, responseConsole); 124 | const response = await page.goto(url, {waitUntil: 'domcontentloaded'}); 125 | 126 | console.log(`Waiting for DOM to settle: ${url}`); 127 | const start = Date.now(); 128 | await waitForDomToSettle(page).catch(e => { 129 | console.warn('Timed out while waiting for DOM', e); 130 | }); 131 | console.log(`Waited ${((Date.now() - start) / 1_000).toFixed(1) }s for ${url} to render.`); 132 | 133 | const html = await page.content(); 134 | 135 | const responseHeaders = response?.headers() ?? {}; 136 | const statusCode = response?.status() ?? 400; 137 | if (response && !responseHeaders['content-type']?.includes('text/html')) { 138 | const buffer = await response.body() as Uint8Array; // TODO(acorn1010): Why is this cast necessary? 139 | return {buffer, statusCode, console: responseConsole, headers: responseHeaders}; 140 | } 141 | return {buffer: Buffer.from(html), statusCode, console: responseConsole, headers: responseHeaders}; 142 | } catch (e: any) { 143 | if (e.message.includes('net::ERR_NAME_NOT_RESOLVED') 144 | || e.message.includes('(Page.navigate): Cannot navigate to invalid URL')) { 145 | return {statusCode: 404, console: [], headers: {}, buffer: Buffer.from([])}; 146 | } else if (e.message.includes('net::ERR_ABORTED')) { 147 | console.error(`Unable to render URL: "${url}". Using fallback.`); 148 | // Failed to fetch. Maybe try without the browser this time? 149 | return fetchPage(url, requestHeaders); 150 | } 151 | console.error('Page failed to render. Recreating browser!', e); 152 | this.recreateBrowser().then(() => {}); 153 | throw e; 154 | } finally { 155 | // Swallow page close errors. The browser might have already closed by this point. 156 | page.close().catch(() => {}).then(() => { 157 | console.log(`Page has closed for URL: ${url}.`); 158 | // Note: We leave the context running in case another request comes in from the same user. 159 | }); 160 | } 161 | } 162 | } 163 | 164 | function getContextKey(requestHeaders: Record) { 165 | const { 166 | ['x-request-id']: xRequestId, // Changes every time 167 | 168 | // These change when a user's IP / that user changes. Because bots are distributed, this 169 | // could cause a new context to be made. 170 | ['x-real-ip']: xRealIp, 171 | ['x-forwarded-for']: xForwardedFor, 172 | ['x-original-forwarded-for']: xOriginalForwardedFor, 173 | ['cf-ray']: cfRay, 174 | ['cf-connecting-ip']: cfConnectingIp, 175 | 176 | ...rest 177 | } = requestHeaders; 178 | return JSON.stringify(rest); 179 | } 180 | 181 | const runner = new ChromeBrowserRunnerSingleton(); 182 | export async function render(url: string, requestHeaders: Record): Promise { 183 | return runner.render(url, requestHeaders); 184 | } 185 | 186 | class Refetcher { 187 | private timeout: NodeJS.Timeout | null = null; 188 | constructor(private readonly runner: ChromeBrowserRunnerSingleton) {} 189 | 190 | /** Starts a page refetcher that is responsible for refetching pages that are expiring. */ 191 | start() { 192 | if (this.timeout) { 193 | return; // Already running 194 | } 195 | this.scheduleInterval(); 196 | } 197 | 198 | close() { 199 | if (this.timeout) { 200 | clearInterval(this.timeout); 201 | } 202 | this.timeout = null; 203 | } 204 | 205 | private scheduleInterval() { 206 | this.timeout = setTimeout(async () => { 207 | try { 208 | await this.render(); 209 | } finally { 210 | if (this.timeout) { 211 | this.scheduleInterval(); 212 | } 213 | } 214 | }, 5_000); // Wait 5 seconds before checking for expiring pages. 215 | } 216 | 217 | private async render() { 218 | // Browser is free to do work! Refresh cache that's about to expire. 219 | const userIdUrls = shuffle(await env.redis.url.queryExpiringUrls()); 220 | const userIdToUser = new Map>(); 221 | for (const {userId, url} of userIdUrls) { 222 | if (!this.timeout) { 223 | return; // Timeout was canceled. 224 | } 225 | if (this.runner.getOutstandingRequestsCount() > 0) { 226 | return; // Browser is busy. Wait. 227 | } 228 | 229 | if (!userIdToUser.has(userId)) { 230 | userIdToUser.set( 231 | userId, 232 | await env.redis.user.get(userId, 'ignoredPaths', 'shouldRefreshCache')); 233 | } 234 | // Attempt to acquire a lock. If successful, render the page. 235 | const start = Date.now(); 236 | if (!await env.redis.lock.acquireLock(`render:${userId}|${url}`, PAGE_EXPIRATION_MS)) { 237 | continue; // Unable to establish lock. Someone else is rendering this page. 238 | } 239 | try { 240 | // Successfully acquired lock. We have ~35 seconds to render this page! 241 | // First, check to see if this page has enough usage. If not, delete it and let it expire. 242 | const reason = await maybeGetRenderSkipReason(url, userId, userIdToUser.get(userId)!); 243 | if (reason) { 244 | console.log(`Skipping render for user: ${userId}`, reason); 245 | env.redis.url.deleteExpiringUrl(userId, url).catch((e) => { 246 | console.error('Failed to delete expiring URL', userId, url, e); 247 | }); 248 | continue; 249 | } 250 | 251 | console.log('Re-rendering before cache expiration', userId, url); 252 | await renderAndCache(userId, url, {}); 253 | } finally { 254 | // If we still hold the lock, clean it up. Small race condition possible here, but I don't 255 | // want to make a Lua script just for this, so w/e. 256 | if (start + PAGE_EXPIRATION_MS > Date.now()) { 257 | env.redis.lock.removeLock(`render:${userId}|${url}`).catch((e) => { 258 | console.error('Failed to remove lock', e); 259 | }); 260 | } 261 | } 262 | } 263 | } 264 | } 265 | 266 | /** Returns a non-empty string if `url` shouldn't be rendered. */ 267 | async function maybeGetRenderSkipReason(url: string, userId: string, user: Pick): Promise { 268 | if (!user.shouldRefreshCache) { 269 | return `Not rendering because shouldNotRefreshCache: "${url}"`; 270 | } 271 | for (const ignoredPath of user.ignoredPaths) { 272 | if (minimatch(url, ignoredPath)) { 273 | return `Not rendering because matched ignorePath: "${url}", "${ignoredPath}"`; 274 | } 275 | } 276 | 277 | const renderCount = await env.redis.url.queryRenderCount(userId, url); 278 | if (renderCount < 2) { 279 | return `Not rendering because too low renderCount: "${url}", ${renderCount}`; 280 | } 281 | 282 | return ''; 283 | } 284 | 285 | export const refetcher = new Refetcher(runner); 286 | -------------------------------------------------------------------------------- /server/src/db/RedisWrapper.ts: -------------------------------------------------------------------------------- 1 | import Redis from "ioredis"; 2 | import {UserModel} from "./models/UserModel"; 3 | import {UrlModel} from "./models/UrlModel"; 4 | import {LockModel} from "@server/db/models/LockModel"; 5 | 6 | // NOTE: We separate the internal and external classes because we don't want to expose the internal 7 | // model interfaces. This is because the internal interfaces are more specific, and we want to be 8 | // able to change them without breaking the external interface. 9 | abstract class RedisWrapperInternal { 10 | protected constructor( 11 | redis: Redis, 12 | 13 | /** Provides an interface for interacting with URLs for a user. */ 14 | readonly url = new UrlModel(redis), 15 | 16 | /** Provides an interface for getting / setting a User's profile / config information. */ 17 | readonly user = new UserModel(redis), 18 | 19 | /** Provides a locking mechanism for workers. */ 20 | readonly lock = new LockModel(redis), 21 | ) {} 22 | } 23 | 24 | export class RedisWrapper extends RedisWrapperInternal { 25 | constructor(redis: Redis) { 26 | super(redis); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /server/src/db/lua/deletePattern.lua: -------------------------------------------------------------------------------- 1 | local cursor = "0" 2 | local pattern = ARGV[1] 3 | local count = 1000 4 | local keysDeleted = 0 5 | 6 | repeat 7 | local res = redis.call("SCAN", cursor, "MATCH", pattern, "COUNT", count) 8 | cursor = res[1] 9 | local keys = res[2] 10 | 11 | if #keys > 0 then 12 | redis.call("DEL", unpack(keys)) 13 | end 14 | keysDeleted = keysDeleted + #keys 15 | until cursor == "0" 16 | 17 | return keysDeleted 18 | -------------------------------------------------------------------------------- /server/src/db/lua/index.ts: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | 3 | /** Imports a script file from Lua. */ 4 | function importFile(path: `./${string}.lua`): string { 5 | return fs.readFileSync(require.resolve(path)).toString('utf8'); 6 | } 7 | 8 | /** 9 | * Deletes cached URLs matching the pattern in ARGV[1] 10 | * 11 | * KEYS: 12 | * hashtag - users:$userId 13 | * ARGS: 14 | * pattern - the URL pattern to delete 15 | */ 16 | export const DELETE_PATTERN = importFile('./deletePattern.lua'); 17 | 18 | /** Refreshes the user's API key. */ 19 | export const REFRESH_USER_TOKEN = importFile('./refreshToken.lua'); 20 | -------------------------------------------------------------------------------- /server/src/db/lua/refreshToken.lua: -------------------------------------------------------------------------------- 1 | local function generate_api_key(userId, length) 2 | local charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" 3 | local key = "" 4 | local seed = tonumber(redis.call('TIME')[1]) 5 | 6 | -- Get the current time as a string. TIME returns [seconds, microseconds] 7 | local time = redis.call('TIME') 8 | local seedStr = userId .. "-" .. time[1] .. time[2] 9 | -- Hash the seed string and convert to a number 10 | local seed = tonumber(redis.sha1hex(seedStr):sub(1, 8), 16) 11 | math.randomseed(seed) 12 | 13 | for i = 1, length do 14 | local random_index = math.random(1, #charset) 15 | key = key .. string.sub(charset, random_index, random_index) 16 | end 17 | return key 18 | end 19 | 20 | local userId = ARGV[1] 21 | 22 | local oldToken = redis.call("HGET", "users:" .. userId, "token") 23 | 24 | if oldToken then 25 | redis.call("HDEL", "tokens", cjson.decode(oldToken)) 26 | end 27 | 28 | -- Create a new token 29 | while true do 30 | local apiKey = generate_api_key(userId, 24) 31 | -- This API key isn't in use. Go ahead and set it! 32 | if not redis.call("HGET", "tokens", apiKey) then 33 | redis.call("HSET", "tokens", apiKey, userId) 34 | redis.call("HSET", "users:" .. userId, "token", cjson.encode(apiKey)) 35 | return apiKey 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /server/src/db/models/AbstractHashModel.ts: -------------------------------------------------------------------------------- 1 | import Redis from "ioredis"; 2 | import {isNil} from "lodash"; 3 | import {Keys, Pretty} from "@shared/types/ExtraTypes"; 4 | 5 | /** An abstract model stores in a Redis hash. */ 6 | export abstract class AbstractHashModel { 7 | protected constructor( 8 | protected readonly redis: Redis, 9 | protected readonly collectionId: string, 10 | private readonly defaultModel: T) {} 11 | 12 | /** 13 | * Queries a model with the given `id` from the database. If `fields` are provided, only those 14 | * fields will be returned. 15 | * 16 | * Complexity is O(n) where `n` is the number of fields requested. 17 | * TODO(acorn1010): Fix return type when key / fields are missing. 18 | */ 19 | async get>(id: string, ...fields: K[]) 20 | : Promise>>> { 21 | if (fields.length === 0) { 22 | const result = await this.redis.hgetall(`${this.collectionId}:${id}`); 23 | return Object.fromEntries( 24 | Object.entries(result).map(([key, value]) => [key, JSON.parse(value)])) as any; 25 | } 26 | 27 | const result = await this.redis.hmget(`${this.collectionId}:${id}`, ...fields); 28 | return Object.fromEntries( 29 | result.map((value, idx) => [fields[idx], isNil(value) ? this.defaultModel[fields[idx]] : JSON.parse(value)]) 30 | ) as any; 31 | } 32 | 33 | /** 34 | * Queries a model for the given `ids` from the database. If `fields` are provided, only those 35 | * fields will be returned. 36 | * 37 | * Complexity is O(n * m) where `n` is the number of fields requested and `m` is the number of `ids`. 38 | */ 39 | async getMany>(ids: string[], ...fields: K[]) 40 | : Promise<(keyof T extends K ? T : Pretty>>)[]> { 41 | return Promise.all(ids.map(id => this.get(id, ...fields))); 42 | } 43 | 44 | /** Updates a model with `id`, setting all field `values`. */ 45 | async update(id: string, values: Partial): Promise { 46 | this.redis.hmset(`${this.collectionId}:${id}`, Object.entries(values).flat()); 47 | } 48 | 49 | /** Creates a new `model` in the database, returning its id. */ 50 | async create(model: T): Promise { 51 | const id = await this.redis.incr(`${this.collectionId}_:id`); 52 | await this.redis.hmset(`${this.collectionId}:${id}`, Object.entries(model).flat()); 53 | return id; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /server/src/db/models/LockModel.ts: -------------------------------------------------------------------------------- 1 | import Redis from "ioredis"; 2 | import {nanoid} from "nanoid"; 3 | 4 | /** A unique id for this process. Ensures that we don't conflict with other worker processes. */ 5 | const uuid = nanoid(); 6 | 7 | export class LockModel { 8 | constructor(private readonly redis: Redis) {} 9 | 10 | /** 11 | * Attempts to acquire a lock for `key` for `expirationMs` milliseconds, returning `true` on 12 | * success. 13 | */ 14 | async acquireLock(key: string, expirationMs: number) { 15 | const ref = `workers:${key}`; 16 | const result = await this.redis.set(ref, uuid, 'NX' as any, 'PX' as any, expirationMs as any); 17 | return result === 'OK'; 18 | } 19 | 20 | /** Removes a lock with the given `key` that was acquired by #acquireLock earlier. */ 21 | async removeLock(key: string) { 22 | this.redis.del(`workers:${key}`); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /server/src/db/models/UrlModel.ts: -------------------------------------------------------------------------------- 1 | import Redis from "ioredis"; 2 | import {promisify} from "util"; 3 | import zlib from "zlib"; 4 | import {DELETE_PATTERN} from "../lua"; 5 | import * as Url from "url"; 6 | import {getYyyyMm, getYyyyMmDd} from "../../TimeUtils"; 7 | import {ConsoleMessageType} from "../../browsers/BrowserUtils"; 8 | 9 | const decompressBrotli = promisify(zlib.brotliDecompress); 10 | const compressBrotli = promisify(zlib.brotliCompress); 11 | 12 | // Cache every hour. Reduce to once per day later on 13 | const CACHE_TIME_MS = 60 * 60 * 1_000; 14 | 15 | /** Maximum amount of time in ms to fetch a URL before it expires. */ 16 | const REFETCH_BUFFER_MS = 15 * 60_000; 17 | 18 | export type RenderResponse = { 19 | renderTimeMs: number, 20 | headers: Record, 21 | /** Log of the console while rendering the request. */ 22 | console: {type: ConsoleMessageType, args: string[]}[], 23 | /** Response status code (e.g. 200 for success) */ 24 | statusCode: 200 | 404 | number, 25 | /** The HTML / binary content of this page */ 26 | buffer: Uint8Array, 27 | }; 28 | 29 | export class UrlModel { 30 | constructor(private readonly redis: Redis) {} 31 | 32 | /** Flushes all the URLs for `userId`. Useful when redeploying a site. */ 33 | async flush(userId: string): Promise { 34 | const result = this.redis.eval(DELETE_PATTERN, 1, `users:${userId}`, `users:${userId}:urls:*`); 35 | return parseInt(result as any); 36 | } 37 | 38 | /** 39 | * Deletes a URL from the expiring URLs collection. Deleting the URL will stop any refetcher from 40 | * trying to refetch this URL. 41 | */ 42 | async deleteExpiringUrl(userId: string, url: string) { 43 | await this.redis.zrem('urlExpiresAt', `${userId}|${url}`); 44 | } 45 | 46 | /** Returns URLs that are about to expire. Used for re-caching. */ 47 | async queryExpiringUrls(): Promise<{userId: string, url: string}[]> { 48 | const maxMs = Date.now() + REFETCH_BUFFER_MS; 49 | const userIdUrls = await this.redis.zrangebyscore('urlExpiresAt', 0, maxMs, 'LIMIT', 0, 1_000); 50 | 51 | return userIdUrls.map(userIdUrl => { 52 | const index = userIdUrl.indexOf('|'); 53 | const userId = index >= 0 ? userIdUrl.slice(0, index) : 'jellybean'; // FIXME(acorn1010): Delete this fake username once we have auth. 54 | const url = userIdUrl.slice(index + 1); 55 | return {userId, url}; 56 | }); 57 | } 58 | 59 | /** Returns the number of times this page has been rendered in the past 2 months. */ 60 | async queryRenderCount(userId: string, url: string): Promise { 61 | return (await Promise.all([ 62 | this.redis.zscore(`users:${userId}:fetches:${getYyyyMm(-1)}`, url), 63 | this.redis.zscore(`users:${userId}:fetches:${getYyyyMm()}`, url), 64 | ])).map(value => +(value || 0)).reduce((a, b) => a + b, 0); 65 | } 66 | 67 | /** 68 | * Returns a cached page for `url` if it exists for `userId`. Increments the count of `userAgent` 69 | * that have requested this page. 70 | */ 71 | async queryPage({userId, url, userAgent}: {userId: string, url: string, userAgent?: string}): Promise { 72 | const key = `users:${userId}:urls:${urlToKey(url)}`; 73 | const yyyyMm = getYyyyMm(); 74 | const [metadata, data] = 75 | (await this.redis.multi() 76 | .get(`${key}:m`) 77 | .getBuffer(`${key}:d`) 78 | .zincrby(`${key}:u`, 1, userAgent || '_') 79 | .pexpire(`${key}:u`, 30 * 24 * 60 * 60 * 1_000) 80 | .zincrby(`users:${userId}:fetches:${yyyyMm}`, 1, url) 81 | .pexpire(`users:${userId}:fetches:${yyyyMm}`, 365 * 24 * 60 * 60 * 1_000/*, 'NX'*/) // NOTE: 'NX' is supported as of v7. Update as soon as it's stable 82 | .exec()) as any as [[null, RenderResponse], [null, Buffer]]; 83 | if (!metadata?.[1] || !data?.[1]) { 84 | return null; // Missing either metadata or the page content itself 85 | } 86 | return { 87 | ...JSON.parse(metadata[1] as any), 88 | // TODO(acorn1010): Remove when everything is Brotli'd 89 | buffer: await decompressBrotli(data[1]).catch(() => data[1]), 90 | }; 91 | } 92 | 93 | /** Sets the page content / metadata of a `url`. */ 94 | async setPage({userId, url, renderResponse}: {userId: string, url: string, renderResponse: RenderResponse}) { 95 | const now = Date.now(); 96 | const {buffer, ...rest} = renderResponse; 97 | const statusCode = renderResponse.statusCode; 98 | const key = `users:${userId}:urls:${urlToKey(url)}`; 99 | const yyyyMm = getYyyyMm(); 100 | const yyyyMmDd = getYyyyMmDd(); 101 | compressBrotli(Buffer.from(buffer)).then(compressed => { 102 | const commander = this.redis.multi() 103 | .incr(`users:${userId}:renderCounts:${yyyyMm}`) // Number of times user has rendered a page 104 | .incr(`users:${userId}:renderCounts:${yyyyMmDd}`) // Number of times user has rendered a page 105 | .pexpire(`users:${userId}:renderCounts:${yyyyMm}`, 365 * 24 * 60 * 60 * 1_000/*, 'NX'*/) 106 | .set(`${key}:m`, JSON.stringify(rest), 'PX', CACHE_TIME_MS) 107 | .setBuffer(`${key}:d`, compressed, 'PX' as any, CACHE_TIME_MS as any); 108 | // If this request succeeded, then log when it expires so we can refresh it before cache 109 | // expiration. 110 | if (statusCode < 400) { 111 | commander.zadd('urlExpiresAt', now + CACHE_TIME_MS, `${userId}|${url}`); 112 | } 113 | commander.exec(); 114 | }); 115 | } 116 | } 117 | 118 | /** 119 | * Given a URL (e.g. "https://api.foony.com/foo/bar"), returns a nice key sorted by TLD, e.g.: 120 | * "https://com:foony:api/foo/bar" 121 | */ 122 | function urlToKey(url: string): string { 123 | const {protocol, host, path} = Url.parse(url); 124 | 125 | // `host` will be the domain name + ':port' (if there's a port), so remove the port, 126 | // reverse the order of the sections, then stick it back together 127 | const [hostname, port] = (host || '').split(':'); 128 | const reverseHostname = hostname.split('.').reverse().join(':'); 129 | return `${protocol}${reverseHostname}${port ? `:${port}` : ''}${path}`; 130 | } 131 | -------------------------------------------------------------------------------- /server/src/db/models/UserModel.ts: -------------------------------------------------------------------------------- 1 | import Redis from "ioredis"; 2 | import {range} from "lodash"; 3 | import {getYyyyMm} from "../../TimeUtils"; 4 | import {User} from "@shared/models/User"; 5 | import {REFRESH_USER_TOKEN} from "../lua"; 6 | import {AbstractHashModel} from "@server/db/models/AbstractHashModel"; 7 | 8 | export class UserModel extends AbstractHashModel { 9 | constructor(redis: Redis) { 10 | super(redis, 'users', { 11 | /** Wildcard to match the page title for 404 pages. These pages will return a `404` status code. */ 12 | wildcard404: '', 13 | 14 | /** 15 | * If `true`, the pages that were fetched in the last month will be refreshed right before they 16 | * expire from the cache. 17 | */ 18 | shouldRefreshCache: false, 19 | 20 | /** 21 | * Wildcard patterns, such as "foo.com/games/*". You shouldn't specify the http:// prefix. If a 22 | * pattern starts with "*" followed by "/", then it will apply to all domains. 23 | */ 24 | ignoredPaths: [], 25 | 26 | /** The user's private API key. Used to make requests via Cloudflare Workers / HTTPS. */ 27 | token: '', 28 | }); 29 | } 30 | 31 | /** Given a user token, returns the userId for that token, or `null` if not found. */ 32 | async getUserIdByToken(token: string): Promise { 33 | return this.redis.hget('tokens', token); 34 | } 35 | 36 | /** Returns the number of renders this user has done by month. */ 37 | async getMonthlyRenderCounts(userId: string): Promise<{month: string, renderCount: number}[]> { 38 | // We store renderCounts for the past 12 months, so go ahead and figure out all keys we need to 39 | // query. 40 | const months = range(-12, 1).map(monthOffset => getYyyyMm(monthOffset)); 41 | const result = await this.redis.mget(months.map(month => `users:${userId}:renderCounts:${month}`)); 42 | return result.map((count, i) => ({month: months[i], renderCount: count === null ? 0 : parseInt(count)})); 43 | } 44 | 45 | /** Expires the old user token and generates a new one to use. */ 46 | async refreshToken(userId: string): Promise { 47 | return this.redis.eval(REFRESH_USER_TOKEN, 0, userId) as Promise; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /server/src/doCall.ts: -------------------------------------------------------------------------------- 1 | import {FastifyReply, FastifyRequest} from "fastify"; 2 | import {Actions} from "@shared/Action"; 3 | import {env} from "./Environment"; 4 | import {HttpsError} from "./http/HttpsError"; 5 | import {ServerActions} from "./api"; 6 | import {isArray, isBoolean, isObject, isString, map, mapValues} from "lodash"; 7 | import {CallableContext} from "./http/CallableContext"; 8 | 9 | type FastifyActionTypes = 10 | {[K in keyof Actions]: {Body: {a: keyof Actions, d: Actions[keyof Actions]['input']}, Reply: Actions[K]['output']}}; 11 | export async function doCall( 12 | req: FastifyRequest, 13 | res: FastifyReply, 14 | context: CallableContext) { 15 | const {a: action, d: data} = req.body; 16 | const validator = env.validators.get(action); 17 | if (!validator) { 18 | console.warn('Missing validator:', action, context); 19 | throw new HttpsError('failed-precondition', 'Bad request'); 20 | } else if (!validator(data)) { 21 | console.dir('Bad client request', validator.errors); 22 | throw new HttpsError('failed-precondition', "Bad client request. Try refreshing your browser's cache."); 23 | } else if (context.authType === 'token' && !['flush'].includes(action)) { 24 | console.warn('Bad API request:', action, context); 25 | throw new HttpsError('failed-precondition', "This type of request isn't allowed by API token."); 26 | } 27 | 28 | const result = await (ServerActions[action] as any)(context, data); 29 | res.status(200).send(JSON.stringify({d: encode(result)})); 30 | } 31 | 32 | /** 33 | * Encodes arbitrary data in our special format for JSON. 34 | * 35 | * "Inspired" by Firebase's encode method, but with extra support for removing Promise types which 36 | * are used internally by the server to guarantee order of execution while returning quickly to the 37 | * user. 38 | */ 39 | export function encode(paramData: any): any { 40 | let data = paramData; 41 | if (data === null || data === undefined) { 42 | return null; 43 | } 44 | // Oddly, isFinite(new Number(x)) always returns false, so unwrap Numbers. 45 | if (data instanceof Number) { 46 | data = data.valueOf(); 47 | } 48 | if (isFinite(data)) { 49 | // Any number in JS is safe to put directly in JSON and parse as a double 50 | // without any loss of precision. 51 | return data; 52 | } 53 | if (isBoolean(data)) { 54 | return data; 55 | } 56 | if (isString(data)) { 57 | return data; 58 | } 59 | if (isArray(data)) { 60 | return map(data, encode); 61 | } 62 | if (isObject(data)) { 63 | if (data instanceof Promise) { 64 | return null; 65 | } 66 | // It's not safe to use forEach, because the object might be 'array-like' 67 | // if it has a key called 'length'. Note that this intentionally overrides 68 | // any toJSON method that an object may have. 69 | return mapValues(data, encode); 70 | } 71 | // If we got this far, the data is not encodable. 72 | console.error('Data cannot be encoded in JSON.', data); 73 | throw new Error('Data cannot be encoded in JSON: ' + data); 74 | } 75 | -------------------------------------------------------------------------------- /server/src/http/AuthUtils.ts: -------------------------------------------------------------------------------- 1 | import admin from "firebase-admin"; 2 | import NodeCache from "node-cache"; 3 | import {HttpsError} from "./HttpsError"; 4 | import {CallableContext} from "./CallableContext"; 5 | 6 | function assertAuthenticated(context: CallableContext): asserts context is Omit & {uid: string} { 7 | if (!context.uid) { 8 | throw new HttpsError('failed-precondition', 'This endpoint must be called while authenticated.'); 9 | } 10 | } 11 | 12 | /** Returns the user id of the authorized user in "context", else throws an HttpsError. */ 13 | export function getUserId(context: CallableContext) { 14 | assertAuthenticated(context); 15 | return context.uid; 16 | } 17 | 18 | const tokenCache = new NodeCache({stdTTL: 60 * 60, useClones: false}); 19 | /** 20 | * Attempts to very quickly verify an idToken. This should always be done before awaiting on 21 | * verifyIdToken for performance reasons. 22 | * 23 | * Note that this _does_ currently allow expired tokens (up to 60 minutes expired). But whatever. 24 | */ 25 | export function verifyIdTokenFromCache(token: string) { 26 | const result = tokenCache.get(token); 27 | if (result instanceof HttpsError) { 28 | throw result; 29 | } 30 | return result; 31 | } 32 | 33 | /** 34 | * Verifies a Firebase JWT auth token. 35 | * TODO(acorn1010): auth.verifyIdToken() used to be slow. If we have issues, borrow fast version from Foony. 36 | * @param token a JWT token to verify. 37 | * @returns verified user or throws HttpsError on unauthenticated. 38 | */ 39 | export async function verifyIdToken(token: string): Promise { 40 | const result = await admin.auth().verifyIdToken(token); 41 | tokenCache.set(token, result); 42 | return result; 43 | } 44 | -------------------------------------------------------------------------------- /server/src/http/CallableContext.ts: -------------------------------------------------------------------------------- 1 | export type CallableContext = { 2 | /** The authenticated userId, if any. */ 3 | uid?: string, 4 | 5 | /** Type of auth for uid. 'token' is more restricted in what it can access than 'oauth'. */ 6 | authType: 'token' | 'oauth', 7 | }; 8 | -------------------------------------------------------------------------------- /server/src/http/HttpsError.ts: -------------------------------------------------------------------------------- 1 | 2 | const TYPE_TO_STATUS_CODE = { 3 | /** User isn't authenticated (bad login). */ 4 | 'unauthenticated': 401, 5 | 6 | /** 7 | * User is authenticated, but not allowed to access this content (e.g. "user" trying to access 8 | * "admin" console). 9 | */ 10 | 'forbidden': 403, 11 | 12 | /** User provided bad input data. */ 13 | 'failed-precondition': 412, 14 | 15 | /** Internal server error. Happens if an exception is unexpectedly thrown. */ 16 | 'internal': 500, 17 | } as const; 18 | 19 | export class HttpsError extends Error { 20 | constructor(private readonly type: keyof typeof TYPE_TO_STATUS_CODE, message: string) { 21 | super(message); 22 | } 23 | 24 | getHttpErrorCode() { 25 | return TYPE_TO_STATUS_CODE[this.type]; 26 | } 27 | 28 | toJson() { 29 | return `${this.type}: ${this.message}`; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /server/src/http/HttpsUtils.ts: -------------------------------------------------------------------------------- 1 | import {CallableContext} from "./CallableContext"; 2 | import {verifyIdToken, verifyIdTokenFromCache } from "./AuthUtils"; 3 | import {HttpsError} from "./HttpsError"; 4 | import {FastifyReply, FastifyRequest} from "fastify"; 5 | import {env} from "../Environment"; 6 | 7 | /** 8 | * Attempts to retrieve the authorization context from the provided `authorization` header. The 9 | * `authorization` header should be of the form "Bearer jwtTokenHere". If it 10 | * doesn't start with 'Bearer ', then it's assumed to be 'jwtTokenHere' instead. 11 | */ 12 | export async function getAuthContext(authorization: string): Promise { 13 | const [, idToken] = (authorization || '').match(/^Bearer (.*)$/) || [null, authorization]; 14 | if (!idToken) { 15 | // noinspection ExceptionCaughtLocallyJS This is how Firebase does it. 16 | throw new HttpsError('unauthenticated', 'Unauthenticated. No Bearer token.'); 17 | } 18 | try { 19 | const token = verifyIdTokenFromCache(idToken) || await verifyIdToken(idToken); 20 | return {uid: token.uid, authType: 'oauth'}; 21 | } catch (err) { 22 | console.warn('Failed to authenticate request.'); 23 | // noinspection ExceptionCaughtLocallyJS This is how Firebase does it. 24 | throw new HttpsError('unauthenticated', 'Unauthenticated'); 25 | } 26 | } 27 | 28 | type Request = FastifyRequest<{ 29 | Params: {token?: string}, 30 | Headers: {['x-prerender-token']?: string}, 31 | }>; 32 | /** Creates a wrapped Fastify request that performs user validation and handles errors. */ 33 | export const makeWrappedRequest = (callback: (req: FastifyRequest, res: FastifyReply, context: CallableContext) => any) => 34 | async (req: Request, res: FastifyReply) => { 35 | const token: string | any = req.headers['x-prerender-token'] || req.params.token; 36 | let context: CallableContext = {uid: undefined, authType: 'token'}; 37 | if (typeof token === 'string') { 38 | context.uid = (await env.redis.user.getUserIdByToken(token)) ?? undefined; 39 | if (!context.uid) { 40 | throw new HttpsError('unauthenticated', 'Bad API token.'); 41 | } 42 | } 43 | 44 | const authorization = req.headers.authorization; 45 | if (authorization) { 46 | context = await getAuthContext(authorization); 47 | if (!context.uid) { 48 | console.log(`Bad bearer token: ${authorization}`); 49 | throw new HttpsError('unauthenticated', 'Bad Bearer token'); 50 | } 51 | } 52 | 53 | try { 54 | await callback(req, res, context); 55 | } catch (e: any) { 56 | let err = e; 57 | if (!(err instanceof HttpsError)) { 58 | console.error('Failed to run API call.', req.body, e); 59 | err = new HttpsError('internal', 'INTERNAL'); 60 | } 61 | console.error('Error during request.', err.toJson()); 62 | res.status(err.getHttpErrorCode()).send(JSON.stringify({e: err.toJson()})); 63 | } 64 | }; 65 | -------------------------------------------------------------------------------- /server/src/index.ts: -------------------------------------------------------------------------------- 1 | // "Hello Cornelia Cornington, nicknamed Corny. You might be wondering why I wrote this comment. Well Idk either, I just 2 | // wanted to put something funny at the top of the index.ts so every time you open it, you have to use a few seconds of 3 | // your life to appreciate the little things in life. Like this comment. Like and Subscribe. 4 | // -Raf" 5 | import Fastify from 'fastify'; 6 | import {doCall} from "./doCall"; 7 | import {doRequest} from "./api/render"; 8 | import {refetcher} from "./browsers/ChromeBrowser"; 9 | import cors from '@fastify/cors'; 10 | import admin from "firebase-admin"; 11 | import {makeWrappedRequest} from "./http/HttpsUtils"; 12 | 13 | // Initialize Firebase 14 | admin.initializeApp({ 15 | projectId: 'render-1010', 16 | }); 17 | 18 | const server = Fastify(); 19 | server.register(cors, { 20 | origin: '*', 21 | methods: ['GET', 'POST'], 22 | }); 23 | server.post('/api', makeWrappedRequest(doCall)); 24 | server.get('*', makeWrappedRequest(doRequest)); 25 | server.listen({ 26 | host: '0.0.0.0', 27 | port: 3_000, 28 | }).then(() => console.log('Server is up and running!')); 29 | 30 | // Only start the refetcher in prod 31 | if (process.env.PROD) { 32 | console.log('Starting refetcher service!'); 33 | refetcher.start(); 34 | } 35 | -------------------------------------------------------------------------------- /server/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "baseUrl": ".", 4 | "module": "CommonJS", 5 | "noImplicitReturns": true, 6 | "noUnusedLocals": true, 7 | "outDir": "lib", 8 | "rootDirs": ["src", "../shared/src"], 9 | "types": ["node"], 10 | "sourceMap": true, 11 | "strict": true, 12 | "declaration": true, 13 | "declarationMap": true, 14 | "declarationDir": "lib", 15 | "resolveJsonModule": true, 16 | "target": "ES2021", 17 | "esModuleInterop": true, 18 | "skipLibCheck": true, 19 | "paths": { 20 | "@server/*": ["src/*"], 21 | "@shared/*": ["../shared/src/*"] 22 | } 23 | }, 24 | "compileOnSave": true, 25 | "include": [ 26 | "src", 27 | "../shared/src" 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /shared/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "render-shared-library", 3 | "version": "1.0.0", 4 | "license": "GPL-3.0-only", 5 | "description": "Shared library between /server and /client.", 6 | "author": "Acorn1010", 7 | "main": "lib/index.js", 8 | "types": "lib/index.d.ts", 9 | "files": [ 10 | "lib/*" 11 | ], 12 | "scripts": { 13 | "lint": "tslint --project tsconfig.json", 14 | "prebuild": "./node_modules/.bin/ts-json-schema-generator --tsconfig './tsconfig.json' --path 'src/Action.ts' > src/api_schema.json", 15 | "build": "tsc", 16 | "deploy": "npm i && npm run build && npm link", 17 | "test": "echo \"Error: no test specified\" && exit 1" 18 | }, 19 | "dependencies": { 20 | "moderndash": "^3.1.0" 21 | }, 22 | "devDependencies": { 23 | "@types/node": "^18.15.10", 24 | "ts-json-schema-generator": "^1.2.0", 25 | "typescript": "^5.0.4" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /shared/shared.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /shared/src/Action.ts: -------------------------------------------------------------------------------- 1 | import {User} from "./models/User"; 2 | 3 | /** 4 | * Actions that can be sent to the server via the server API #call function. The value follows a 5 | * special namespacing format. Before the first _ is the functions/src/actions/ filename that will 6 | * have its #run method be executed. After that, the rest is action-specific. For example, game 7 | * actions are of the form "GameAction_{gameId}_{methodName}". 8 | */ 9 | export type Actions = MakeActions<{ 10 | /** Flushes the domains of the user. */ 11 | flush: {input: [], output: boolean}, 12 | 13 | /** Returns the page renders done by month for the user. */ 14 | getMonthlyRenderCounts: {input: [], output: {month: string, renderCount: number}[]}, 15 | 16 | /** Retrieves information about a user's profile. */ 17 | getProfile: {input: [], output: User}, 18 | 19 | /** Refreshes a user's API key, returning their new API key. */ 20 | refreshToken: {input: [], output: string}, 21 | }>; 22 | export type Action = keyof Actions; 23 | /** This type is required by ts-json-schema so we can generate proper */ 24 | export type ActionInputs = {[K in keyof Actions]: Actions[K]['input']}; 25 | 26 | /** Validates Action type and ensures everything follows the correct format. */ 27 | type MakeActions = T; 28 | -------------------------------------------------------------------------------- /shared/src/api_schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "definitions": { 4 | "Action": { 5 | "enum": [ 6 | "flush", 7 | "getMonthlyRenderCounts", 8 | "getProfile", 9 | "refreshToken" 10 | ], 11 | "type": "string" 12 | }, 13 | "ActionInputs": { 14 | "additionalProperties": false, 15 | "description": "This type is required by ts-json-schema so we can generate proper", 16 | "properties": { 17 | "flush": { 18 | "maxItems": 0, 19 | "minItems": 0, 20 | "type": "array" 21 | }, 22 | "getMonthlyRenderCounts": { 23 | "maxItems": 0, 24 | "minItems": 0, 25 | "type": "array" 26 | }, 27 | "getProfile": { 28 | "maxItems": 0, 29 | "minItems": 0, 30 | "type": "array" 31 | }, 32 | "refreshToken": { 33 | "maxItems": 0, 34 | "minItems": 0, 35 | "type": "array" 36 | } 37 | }, 38 | "required": [ 39 | "flush", 40 | "getMonthlyRenderCounts", 41 | "getProfile", 42 | "refreshToken" 43 | ], 44 | "type": "object" 45 | }, 46 | "Actions": { 47 | "additionalProperties": false, 48 | "description": "Actions that can be sent to the server via the server API #call function. The value follows a special namespacing format. Before the first _ is the functions/src/actions/ filename that will have its #run method be executed. After that, the rest is action-specific. For example, game actions are of the form \"GameAction_{gameId}_{methodName}\".", 49 | "properties": { 50 | "flush": { 51 | "additionalProperties": false, 52 | "description": "Flushes the domains of the user.", 53 | "properties": { 54 | "input": { 55 | "maxItems": 0, 56 | "minItems": 0, 57 | "type": "array" 58 | }, 59 | "output": { 60 | "type": "boolean" 61 | } 62 | }, 63 | "required": [ 64 | "input", 65 | "output" 66 | ], 67 | "type": "object" 68 | }, 69 | "getMonthlyRenderCounts": { 70 | "additionalProperties": false, 71 | "description": "Returns the page renders done by month for the user.", 72 | "properties": { 73 | "input": { 74 | "maxItems": 0, 75 | "minItems": 0, 76 | "type": "array" 77 | }, 78 | "output": { 79 | "items": { 80 | "additionalProperties": false, 81 | "properties": { 82 | "month": { 83 | "type": "string" 84 | }, 85 | "renderCount": { 86 | "type": "number" 87 | } 88 | }, 89 | "required": [ 90 | "month", 91 | "renderCount" 92 | ], 93 | "type": "object" 94 | }, 95 | "type": "array" 96 | } 97 | }, 98 | "required": [ 99 | "input", 100 | "output" 101 | ], 102 | "type": "object" 103 | }, 104 | "getProfile": { 105 | "additionalProperties": false, 106 | "description": "Retrieves information about a user's profile.", 107 | "properties": { 108 | "input": { 109 | "maxItems": 0, 110 | "minItems": 0, 111 | "type": "array" 112 | }, 113 | "output": { 114 | "$ref": "#/definitions/User" 115 | } 116 | }, 117 | "required": [ 118 | "input", 119 | "output" 120 | ], 121 | "type": "object" 122 | }, 123 | "refreshToken": { 124 | "additionalProperties": false, 125 | "description": "Refreshes a user's API key, returning their new API key.", 126 | "properties": { 127 | "input": { 128 | "maxItems": 0, 129 | "minItems": 0, 130 | "type": "array" 131 | }, 132 | "output": { 133 | "type": "string" 134 | } 135 | }, 136 | "required": [ 137 | "input", 138 | "output" 139 | ], 140 | "type": "object" 141 | } 142 | }, 143 | "required": [ 144 | "flush", 145 | "getMonthlyRenderCounts", 146 | "getProfile", 147 | "refreshToken" 148 | ], 149 | "type": "object" 150 | }, 151 | "User": { 152 | "additionalProperties": false, 153 | "properties": { 154 | "ignoredPaths": { 155 | "description": "Wildcard patterns, such as \"foo.com/games/*\". You shouldn't specify the http:// prefix. If a pattern starts with \"*\" followed by \"/\", then it will apply to all domains.", 156 | "items": { 157 | "type": "string" 158 | }, 159 | "type": "array" 160 | }, 161 | "shouldRefreshCache": { 162 | "description": "If `true`, the pages that were fetched in the last month will be refreshed right before they expire from the cache.", 163 | "type": "boolean" 164 | }, 165 | "token": { 166 | "description": "The user's private API key. Used to make requests via Cloudflare Workers / HTTPS.", 167 | "type": "string" 168 | }, 169 | "wildcard404": { 170 | "description": "Wildcard to match the page title for 404 pages. These pages will return a `404` status code.", 171 | "type": "string" 172 | } 173 | }, 174 | "required": [ 175 | "wildcard404", 176 | "shouldRefreshCache", 177 | "ignoredPaths", 178 | "token" 179 | ], 180 | "type": "object" 181 | } 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /shared/src/collections/BiMap.ts: -------------------------------------------------------------------------------- 1 | /** A two-way Map that allows O(1) lookups by both key and by value. */ 2 | export class BiMap { 3 | private readonly keysToValues = new Map() 4 | private readonly valuesToKeys = new Map(); 5 | 6 | /** Returns the `value` for the given `key`. */ 7 | get(key: K) { 8 | return this.keysToValues.get(key); 9 | } 10 | 11 | /** Returns the `key` matching `value` if `value` was in the BiMap. */ 12 | getByValue(value: V) { 13 | return this.valuesToKeys.get(value); 14 | } 15 | 16 | /** Returns `true` if `key` was deleted from the BiMap. */ 17 | delete(key: K): boolean { 18 | if (!this.keysToValues.has(key)) { 19 | return false; 20 | } 21 | this.valuesToKeys.delete(this.get(key)!); 22 | this.keysToValues.delete(key); 23 | return true; 24 | } 25 | 26 | /** Returns `true` if `value` was deleted from the BiMap. */ 27 | deleteByValue(value: V): boolean { 28 | if (!this.valuesToKeys.has(value)) { 29 | return false; 30 | } 31 | this.valuesToKeys.delete(value); 32 | this.keysToValues.delete(this.getByValue(value)!); 33 | return true; 34 | } 35 | 36 | set(key: K, value: V) { 37 | this.keysToValues.set(key, value); 38 | this.valuesToKeys.set(value, key); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /shared/src/collections/BiMultimap.ts: -------------------------------------------------------------------------------- 1 | // Note: Consider using a whitelist here instead: future API changes could 2 | // introduce mutable methods that allow changing the values in Set. 3 | type ImmutableSet = Omit, 'add' | 'delete' | 'clear'>; 4 | 5 | /** 6 | * A bi-directional Multimap. Multimaps are similar to maps, but store a set of values for a given 7 | * key. A bi-directional Multimap, then, allows you to retrieve all keys for a given `value`, or all 8 | * values from a given `key`. 9 | */ 10 | export class BiMultimap { 11 | private readonly keys = new Map>(); 12 | private readonly values = new Map>(); 13 | 14 | /** Returns true if `key` is in this multimap. */ 15 | has(key: K): boolean { 16 | return this.keys.has(key); 17 | } 18 | 19 | /** Returns the `value` for the given `key`. */ 20 | get(key: K): ImmutableSet | undefined { 21 | return this.keys.get(key); 22 | } 23 | 24 | /** Returns the `key` matching `value` if `value` was in the BiMultimap. */ 25 | getByValue(value: V): ImmutableSet | undefined { 26 | return this.values.get(value); 27 | } 28 | 29 | /** Removes a key-value pair from the multimap, returning `true` if a pair was deleted. */ 30 | delete(key: K, value: V): boolean { 31 | const result = !!this.keys.get(key)?.has(value); 32 | this.keys.get(key)?.delete(value); 33 | this.values.get(value)?.delete(key); 34 | return result; 35 | } 36 | 37 | /** Returns `true` if `key` was deleted from the BiMultimap. */ 38 | deleteByKey(key: K): boolean { 39 | if (!this.keys.has(key)) { 40 | return false; 41 | } 42 | // Delete from `values` all references to `key` 43 | for (const value of this.get(key) || []) { 44 | const valueKeys = this.values.get(value); 45 | if (!valueKeys) { 46 | continue; 47 | } 48 | for (const valueKey of valueKeys) { 49 | valueKeys.delete(valueKey); 50 | } 51 | if (valueKeys.size <= 0) { 52 | this.values.delete(value); 53 | } 54 | } 55 | this.keys.delete(key); 56 | return true; 57 | } 58 | 59 | /** Returns `true` if `value` was deleted from the BiMultimap. */ 60 | deleteByValue(value: V): boolean { 61 | if (!this.values.has(value)) { 62 | return false; 63 | } 64 | // Delete from `keys` all references to `value` 65 | for (const key of this.getByValue(value) || []) { 66 | const keyValues = this.keys.get(key); 67 | if (!keyValues) { 68 | continue; 69 | } 70 | for (const keyValue of keyValues) { 71 | keyValues.delete(keyValue); 72 | } 73 | if (keyValues.size <= 0) { 74 | this.keys.delete(key); 75 | } 76 | } 77 | this.values.delete(value); 78 | return true; 79 | } 80 | 81 | /** 82 | * Adds `key`, `value` to the multimap, returning `true` if it was added or 83 | * false if the key-value pair already existed. 84 | */ 85 | set(key: K, value: V): boolean { 86 | if (!this.keys.has(key)) { 87 | this.keys.set(key, new Set()); 88 | } 89 | if (!this.values.has(value)) { 90 | this.values.set(value, new Set()); 91 | } 92 | const result = !this.keys.get(key)!.has(value); 93 | this.keys.get(key)!.add(value); 94 | this.values.get(value)!.add(key); 95 | return result; 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /shared/src/collections/CyclicBuffer.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * A cyclic buffer that provides iteration of its values. 3 | * 4 | * NOTE: The buffer can keep around old memory. Be VERY careful when using this with large objects, 5 | * as a reference to those objects may persist past the expected lifecycle. 6 | */ 7 | export class CyclicBuffer { 8 | private readonly buffer: Array; 9 | 10 | /** Index into buffer. This is one past the 0-based value of the last item added to the buffer. */ 11 | private index = 0; 12 | 13 | /** Number of elements currently in the cyclic buffer. */ 14 | public length = 0; 15 | 16 | /** Creates a cyclic buffer with `size` elements in it. */ 17 | constructor(size: number) { 18 | this.buffer = new Array(size); 19 | } 20 | 21 | /** Sets the length of the buffer to 0. Does NOT remove references to objects! */ 22 | clear() { 23 | this.length = 0; 24 | } 25 | 26 | /** Pops the newest value from the buffer, if any. Else returns undefined. */ 27 | popBack(): T | undefined { 28 | if (this.length <= 0) { 29 | return undefined; 30 | } 31 | if (this.index <= 0) { 32 | this.index = this.buffer.length; 33 | } 34 | --this.length; 35 | return this.buffer[--this.index]; 36 | } 37 | 38 | /** Pops the oldest value from the buffer, if any. Else returns undefined. */ 39 | popFront(): T | undefined { 40 | if (this.length <= 0) { 41 | return undefined; 42 | } 43 | let index = this.index - this.length--; 44 | if (index < 0) { 45 | index += this.buffer.length; 46 | } 47 | return this.buffer[index]; 48 | } 49 | 50 | /** Pushes `values` onto the buffer at the back. */ 51 | pushBack(...values: T[]) { 52 | for (const value of (values)) { 53 | this.length = Math.min(this.length + 1, this.buffer.length); 54 | this.buffer[this.index++] = value; 55 | // Reset index if it's too high. 56 | if (this.index >= this.buffer.length) { 57 | this.index = 0; 58 | } 59 | } 60 | } 61 | 62 | /** 63 | * Returns values from most-recently-added to least-recent. Does not modify the buffer. 64 | * See Array#slice. 65 | */ 66 | slice(start: number, end?: number) { 67 | const result: T[] = []; 68 | 69 | start = start < 0 ? this.buffer.length + start : start; 70 | end = end ?? this.buffer.length; 71 | end = end !== undefined && end < 0 ? this.buffer.length + end : end; 72 | const length = end - start; 73 | const index = (this.index - start) % this.buffer.length; 74 | result.push(...this.buffer.slice(Math.max(0, index - length), index).reverse()); 75 | result.push(...this.buffer.slice(this.buffer.length - (length - index)).reverse()); 76 | return result; 77 | } 78 | 79 | /** Returns values in the buffer from newest to oldest. */ 80 | [Symbol.iterator](): Iterator { 81 | const that = this; 82 | let currentIndex = this.index; 83 | let count = 0; 84 | 85 | return { 86 | next(): IteratorResult { 87 | if (currentIndex <= 0) { 88 | currentIndex = that.buffer.length; 89 | } 90 | const value = that.buffer[--currentIndex]; 91 | return {done: count++ === that.buffer.length, value}; 92 | } 93 | }; 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /shared/src/collections/Multiset.ts: -------------------------------------------------------------------------------- 1 | /** A set that can have multiple entries of the same value. Thanks, ChatGPT! */ 2 | export class Multiset { 3 | private readonly elements = new Map(); 4 | /** Total number of elements in the Multiset. */ 5 | private _size = 0; 6 | 7 | /** Returns the number of elements in the set after adding `element`. */ 8 | add(element: T): number { 9 | const count = 1 + (this.elements.get(element) || 0); 10 | this.elements.set(element, count); 11 | ++this._size; 12 | return count; 13 | } 14 | 15 | /** Returns `true` if the element was removed from the set. */ 16 | remove(element: T): boolean { 17 | const currentCount = this.elements.get(element) || 0; 18 | if (currentCount < 1) { 19 | return false; 20 | } 21 | --this._size; 22 | if (currentCount === 1) { 23 | this.elements.delete(element); 24 | } else { 25 | this.elements.set(element, currentCount - 1); 26 | } 27 | return true; 28 | } 29 | 30 | /** Returns `true` if `element` is in the Multiset */ 31 | has(element: T): boolean { 32 | return this.elements.has(element); 33 | } 34 | 35 | /** 36 | * Returns the total number of elements in the Multiset (e.g. adding the same key 3 times counts 37 | * as 3 elements). 38 | */ 39 | size() { 40 | return this._size; 41 | } 42 | 43 | /** Returns an iterator over the values in the Multiset. */ 44 | *[Symbol.iterator](): Iterator { 45 | for (const value of this.elements.keys()) { 46 | yield value; 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /shared/src/constants.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Production feature flags. If true, these flags will be enabled on the server during deployments. 3 | * Most of these should be set to false. Once a feature flag has been `true` for a while (e.g. a 4 | * few weeks), it should be removed. 5 | */ 6 | const PROD_FEATURE_FLAGS = { 7 | } as const; 8 | 9 | /** 10 | * Dev feature flags. These flags will show up during local development. Add your flag here so you 11 | * can see your changes as you're developing a feature. 12 | */ 13 | const DEV_FEATURE_FLAGS = { 14 | } as const; 15 | 16 | /** 17 | * Feature flags. These allow development of a feature while keeping it disabled in the client and 18 | * server. Feature flags should be removed once the feature is fully implemented and live. 19 | */ 20 | export const FEATURE_FLAGS = process.env.NODE_ENV === 'development' ? DEV_FEATURE_FLAGS : PROD_FEATURE_FLAGS; 21 | -------------------------------------------------------------------------------- /shared/src/index.ts: -------------------------------------------------------------------------------- 1 | /** @module shared-library */ 2 | export * from './Action'; 3 | -------------------------------------------------------------------------------- /shared/src/models/User.ts: -------------------------------------------------------------------------------- 1 | export type User = { 2 | /** Wildcard to match the page title for 404 pages. These pages will return a `404` status code. */ 3 | wildcard404: string, 4 | 5 | /** 6 | * If `true`, the pages that were fetched in the last month will be refreshed right before they 7 | * expire from the cache. 8 | */ 9 | shouldRefreshCache: boolean, 10 | 11 | /** 12 | * Wildcard patterns, such as "foo.com/games/*". You shouldn't specify the http:// prefix. If a 13 | * pattern starts with "*" followed by "/", then it will apply to all domains. 14 | */ 15 | ignoredPaths: string[], 16 | 17 | /** The user's private API key. Used to make requests via Cloudflare Workers / HTTPS. */ 18 | token: string, 19 | }; 20 | -------------------------------------------------------------------------------- /shared/src/types/ExtraTypes.ts: -------------------------------------------------------------------------------- 1 | /** Makes `T` readonly for all nested properties. */ 2 | export type DeepReadonly = 3 | T extends Function ? T 4 | : T extends object ? {readonly [key in keyof T]: DeepReadonly} 5 | : T; 6 | 7 | /** Helper type to flatten complex types. */ 8 | export type Pretty = T extends infer U ? {[K in keyof U]: U[K]} : never; 9 | 10 | /** Limits keyof T to extending just string. Sometimes needed so we don't have a string | number | symbol union. */ 11 | export type Keys = Extract; 12 | -------------------------------------------------------------------------------- /shared/src/utils/ArrayUtils.ts: -------------------------------------------------------------------------------- 1 | import {isEqual} from 'moderndash'; 2 | 3 | /** 4 | * Shuffles an "array" in-place and returns the array. Uses the Durstenfeld shuffle.
    5 | * See: https://stackoverflow.com/questions/2450954/how-to-randomize-shuffle-a-javascript-array 6 | */ 7 | export function shuffleArray(array: T[]): T[] { 8 | for (let i = array.length - 1; i > 0; i--) { 9 | const j = Math.floor(Math.random() * (i + 1)); 10 | [array[i], array[j]] = [array[j], array[i]]; 11 | } 12 | return array; 13 | } 14 | 15 | export function last(value: []): undefined; 16 | export function last(value: [...T, L]): L; 17 | /** 18 | * Returns the last element of `values`, or `undefined` if passed an empty list. 19 | * Credit to github.com/tobshub for the really clever last above! 20 | * Credit to keraion for removing the ts-ignore requirement above! 21 | * @param values 22 | */ 23 | export function last(values: T[]): T | undefined; 24 | export function last(values: T[]): T | undefined { 25 | return values[values.length - 1]; 26 | } 27 | 28 | export function deepUnique(values: T[]): T[] { 29 | return values.filter((element, index) => values.findIndex((step) => isEqual(element, step)) === index); 30 | } 31 | -------------------------------------------------------------------------------- /shared/src/utils/NumberUtils.ts: -------------------------------------------------------------------------------- 1 | // noinspection JSUnusedGlobalSymbols This function is used in both the client and server. 2 | /** Clamps a value to be between [min, max] (inclusive). */ 3 | export function clamp(value: number, min: number, max: number) { 4 | return min < max 5 | ? Math.max(min, Math.min(value, max)) 6 | : Math.max(max, Math.min(value, min)); 7 | } 8 | -------------------------------------------------------------------------------- /shared/src/utils/StringUtils.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * A weak hash that, given a `key` and `seed`, returns a 32-bit positive integer hash. 3 | * 4 | * @author Gary Court 5 | * @see http://github.com/garycourt/murmurhash-js 6 | * @author Austin Appleby 7 | * @see http://sites.google.com/site/murmurhash/ 8 | * 9 | * @param {string} key ASCII only 10 | * @param {number} seed Positive integer only 11 | * @return {number} 32-bit positive integer hash 12 | */ 13 | export function weakHash(key: string, seed: number) { 14 | let remainder, bytes, h1, h1b, c1, c2, k1, i; 15 | 16 | remainder = key.length & 3; // key.length % 4 17 | bytes = key.length - remainder; 18 | h1 = Math.abs(seed); 19 | c1 = 0xcc9e2d51; 20 | c2 = 0x1b873593; 21 | i = 0; 22 | 23 | while (i < bytes) { 24 | k1 = 25 | ((key.charCodeAt(i) & 0xff)) | 26 | ((key.charCodeAt(++i) & 0xff) << 8) | 27 | ((key.charCodeAt(++i) & 0xff) << 16) | 28 | ((key.charCodeAt(++i) & 0xff) << 24); 29 | ++i; 30 | 31 | k1 = ((((k1 & 0xffff) * c1) + ((((k1 >>> 16) * c1) & 0xffff) << 16))) & 0xffffffff; 32 | k1 = (k1 << 15) | (k1 >>> 17); 33 | k1 = ((((k1 & 0xffff) * c2) + ((((k1 >>> 16) * c2) & 0xffff) << 16))) & 0xffffffff; 34 | 35 | h1 ^= k1; 36 | h1 = (h1 << 13) | (h1 >>> 19); 37 | h1b = ((((h1 & 0xffff) * 5) + ((((h1 >>> 16) * 5) & 0xffff) << 16))) & 0xffffffff; 38 | h1 = (((h1b & 0xffff) + 0x6b64) + ((((h1b >>> 16) + 0xe654) & 0xffff) << 16)); 39 | } 40 | 41 | k1 = 0; 42 | 43 | // Unwrapped switch statement because TypeScript was complaining about case fallthrough. 44 | if (remainder === 3) { 45 | k1 ^= (key.charCodeAt(i + 2) & 0xff) << 16; 46 | } 47 | if (remainder >= 2) { 48 | k1 ^= (key.charCodeAt(i + 1) & 0xff) << 8; 49 | } 50 | if (remainder >= 1) { 51 | k1 ^= (key.charCodeAt(i) & 0xff); 52 | } 53 | k1 = (((k1 & 0xffff) * c1) + ((((k1 >>> 16) * c1) & 0xffff) << 16)) & 0xffffffff; 54 | k1 = (k1 << 15) | (k1 >>> 17); 55 | k1 = (((k1 & 0xffff) * c2) + ((((k1 >>> 16) * c2) & 0xffff) << 16)) & 0xffffffff; 56 | h1 ^= k1; 57 | 58 | h1 ^= key.length; 59 | 60 | h1 ^= h1 >>> 16; 61 | h1 = (((h1 & 0xffff) * 0x85ebca6b) + ((((h1 >>> 16) * 0x85ebca6b) & 0xffff) << 16)) & 0xffffffff; 62 | h1 ^= h1 >>> 13; 63 | h1 = ((((h1 & 0xffff) * 0xc2b2ae35) + ((((h1 >>> 16) * 0xc2b2ae35) & 0xffff) << 16))) & 0xffffffff; 64 | h1 ^= h1 >>> 16; 65 | 66 | return h1 >>> 0; 67 | } 68 | 69 | /** 70 | * Modeled after base64 web-safe chars, but ordered by ASCII. These are the characters allowed in a 71 | * short push id, such as the one used for all rooms. 72 | */ 73 | export const SHORT_PUSH_ID_CHARS = '123456789abcdefghijkmnopqrstuvwxyz'; 74 | export const SHORT_PUSH_ID_LENGTH = 8; 75 | /** 76 | * Generates a short, Firebase-esque push ID. This is used for roomIds and other areas where the 77 | * push ID is visible to the public and needs to be easy to type. 78 | * 79 | * Unlike a Firebase push ID, the returned ID has no timestamp component and has ~40-bits of 80 | * entropy. This is enough for an 89% chance of no collisions at 1M push ids. The id also does not 81 | * contain commonly-confused characters ("O", "0", "l", "I"). 82 | * 83 | * NOTE: This means the short push IDs are not chronologically ordered. 84 | * 85 | * See: https://instacalc.com/28845 for calculation of probabilities (Birthday paradox). 86 | */ 87 | export function generateShortPushId() { 88 | let id = ''; 89 | 90 | for (let i = 0; i < SHORT_PUSH_ID_LENGTH; ++i) { 91 | id += SHORT_PUSH_ID_CHARS.charAt(Math.floor(Math.random() * SHORT_PUSH_ID_CHARS.length)); 92 | } 93 | if (id.length !== SHORT_PUSH_ID_LENGTH) { 94 | throw new Error(`Short PushID length should be ${SHORT_PUSH_ID_LENGTH}.`); 95 | } 96 | 97 | return id; 98 | } 99 | 100 | /** Lowercases the first character of `value` (e.g. "BOb" -> "bOb") */ 101 | export function lowerFirst(value: string) { 102 | return value ? value.charAt(0).toLowerCase() + value.slice(1) : ''; 103 | } 104 | -------------------------------------------------------------------------------- /shared/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "baseUrl": "src", 4 | "module": "CommonJS", 5 | "noImplicitReturns": true, 6 | "noUnusedLocals": true, 7 | "outDir": "lib", 8 | "rootDir": "src", 9 | "types": ["node"], 10 | "resolveJsonModule": true, 11 | "sourceMap": true, 12 | "strict": true, 13 | "declaration": true, 14 | "declarationMap": true, 15 | "declarationDir": "lib", 16 | "target": "ES2020", 17 | "paths": {"@shared/*": ["src/*"]} 18 | }, 19 | "compileOnSave": true, 20 | "include": [ 21 | "src", 22 | "src/api_schema.json" 23 | ] 24 | } 25 | --------------------------------------------------------------------------------