├── .dockerignore ├── .eslintrc.js ├── .github └── workflows │ └── main.yaml ├── .gitignore ├── Dockerfile ├── NOTES.md ├── README.md ├── assets └── desc_sample.json ├── chart ├── .helmignore ├── Chart.yaml └── templates │ ├── configmap.yaml │ ├── deployment.yaml │ └── service.yaml ├── jest.config.js ├── package.json ├── src ├── database │ ├── helpers │ │ ├── index.ts │ │ └── package.ts │ ├── index.ts │ ├── model │ │ ├── base.ts │ │ ├── index.ts │ │ ├── manifest.ts │ │ ├── package.ts │ │ └── stats.ts │ ├── service │ │ ├── base.ts │ │ ├── index.ts │ │ ├── manifest.ts │ │ ├── package.ts │ │ └── stats.ts │ └── types │ │ ├── base.ts │ │ ├── error.ts │ │ ├── index.ts │ │ ├── manifest.ts │ │ ├── package.ts │ │ ├── service.ts │ │ └── stats.ts ├── env.d.ts ├── index.ts └── server │ ├── buildFastify.ts │ ├── ghService │ ├── helpers │ │ └── decodingHelper.ts │ ├── import │ │ ├── importPackageUtil.ts │ │ ├── manualImportUtil.ts │ │ └── singlePackageImport.ts │ ├── index.ts │ ├── types │ │ ├── import │ │ │ ├── batchImportModel.ts │ │ │ └── manifestFolderListModel.ts │ │ └── update │ │ │ ├── commitDetailsModel.ts │ │ │ ├── masterCommitModel.ts │ │ │ └── packageFileDetailsModel.ts │ └── update │ │ ├── manualPackageUpdateUtil.ts │ │ └── updatePackageUtil.ts │ ├── helpers │ ├── index.ts │ └── validateApiToken.ts │ ├── index.ts │ ├── plugins │ ├── index.ts │ └── ratelimit.ts │ └── routes │ ├── index.ts │ ├── v1 │ └── index.ts │ └── v2 │ ├── featuredPackages.ts │ ├── github.ts │ ├── index.ts │ ├── manifests.ts │ ├── packages.ts │ └── stats.ts ├── tests ├── setup.ts └── test.spec.ts ├── tsconfig.json └── yarn.lock /.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | coverage 4 | .pnp.* 5 | .env 6 | telepresence.log 7 | yarn-error.log 8 | secret.yaml 9 | *.pem 10 | *.pub 11 | temp 12 | dump 13 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | es6: true, 4 | jest: true, 5 | }, 6 | extends: [ 7 | "plugin:@typescript-eslint/recommended", 8 | "airbnb-base", 9 | ], 10 | globals: { 11 | Atomics: "readonly", 12 | SharedArrayBuffer: "readonly", 13 | }, 14 | parser: "@typescript-eslint/parser", 15 | parserOptions: { 16 | ecmaVersion: 2018, 17 | sourceType: "module", 18 | }, 19 | plugins: [ 20 | "@typescript-eslint", 21 | ], 22 | rules: { 23 | quotes: ["error", "double"], 24 | "import/extensions": "off", 25 | "no-console": "off", 26 | "arrow-parens": "off", 27 | "max-len": ["error", 150], 28 | "@typescript-eslint/interface-name-prefix": "off", 29 | // doesnt work with typescript stuff 30 | "no-undef": "off", 31 | "no-bitwise": "off", 32 | // working with mongo _ids 33 | "no-underscore-dangle": "off", 34 | }, 35 | settings: { 36 | "import/resolver": { 37 | node: { 38 | extensions: [".ts"], 39 | }, 40 | }, 41 | }, 42 | }; 43 | -------------------------------------------------------------------------------- /.github/workflows/main.yaml: -------------------------------------------------------------------------------- 1 | name: Check and deploy 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - master 7 | push: 8 | branches: 9 | - master 10 | 11 | env: 12 | # general 13 | GITHUB_SHA: ${{ github.sha }} 14 | GITHUB_ORG: ${{ github.event.organization.login }} 15 | GITHUB_REPO: ${{ github.event.repository.name }} 16 | GITHUB_BRANCH: ${{ github.ref_name }} 17 | 18 | # docker 19 | CONTAINER_REGISTRY: ghcr.io 20 | CONTAINER_IMAGE: ${{ github.event.repository.name }} 21 | 22 | # deployment 23 | K8S_CLUSTER: k8s-bandsy 24 | HELM_RELEASE_NAMESPACE: winget-run-dev 25 | HELM_RELEASE: ${{ github.event.repository.name }} 26 | HELM_CHART_DIR: ./chart 27 | 28 | jobs: 29 | check: 30 | name: General code checks 31 | runs-on: ubuntu-latest 32 | steps: 33 | - uses: actions/checkout@v2 34 | - name: Setup Node 35 | uses: actions/setup-node@v2 36 | with: 37 | node-version: lts/* 38 | - name: Setup pnpm 39 | uses: pnpm/action-setup@v2.0.1 40 | with: 41 | version: 6 42 | - name: Install dependencies 43 | run: pnpm install 44 | - name: Ensure correct formatting 45 | run: pnpm format:check 46 | - name: Ensure correct linting 47 | run: pnpm lint:check 48 | - name: Ensure tests pass requirements 49 | run: pnpm test:check 50 | - name: Ensure code builds 51 | run: pnpm build:check 52 | build: 53 | name: Build container image 54 | if: github.event_name == 'push' 55 | needs: check 56 | runs-on: ubuntu-latest 57 | steps: 58 | - uses: actions/checkout@v2 59 | - name: Build and push image to registry 60 | uses: mr-smithers-excellent/docker-build-push@v5 61 | with: 62 | image: ${{ env.CONTAINER_IMAGE }} 63 | tags: ${{ env.GITHUB_SHA }},${{ env.GITHUB_BRANCH }},latest 64 | githubOrg: ${{ env.GITHUB_ORG }} 65 | registry: ${{ env.CONTAINER_REGISTRY }} 66 | username: ${{ github.actor }} 67 | password: ${{ secrets.GITHUB_TOKEN }} 68 | deploy: 69 | name: Deploy to kubernetes 70 | if: github.event_name == 'push' 71 | needs: build 72 | runs-on: ubuntu-latest 73 | steps: 74 | - uses: actions/checkout@v2 75 | - name: Install doctl 76 | uses: digitalocean/action-doctl@v2 77 | with: 78 | token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} 79 | - name: Save kubeconfig 80 | run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ env.K8S_CLUSTER }} 81 | - name: Install helm 82 | run: |- 83 | curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 84 | chmod 700 get_helm.sh 85 | ./get_helm.sh 86 | - name: Update deployment 87 | run: helm upgrade ${{ env.HELM_RELEASE }} ${{ env.HELM_CHART_DIR }} --install --atomic --namespace ${{ env.HELM_RELEASE_NAMESPACE }} --set-string sha=${{ env.GITHUB_SHA }} 88 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | coverage 4 | .pnp.* 5 | .env 6 | telepresence.log 7 | yarn-error.log 8 | secret.yaml 9 | *.pem 10 | *.pub 11 | *.crt 12 | temp 13 | dump 14 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:12-alpine 2 | 3 | WORKDIR /app 4 | 5 | COPY . . 6 | 7 | RUN yarn 8 | RUN yarn build 9 | 10 | EXPOSE 3000 11 | 12 | ENTRYPOINT ["yarn", "run:prod"] 13 | -------------------------------------------------------------------------------- /NOTES.md: -------------------------------------------------------------------------------- 1 | ## GitHub actions 2 | 3 | Secrets: 4 | - DIGITALOCEAN_ACCESS_TOKEN 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Winget.run API 2 | 3 | The REST API behind [winget.run](https://winget.run), allowing users to search, discover, and install winget packages effortlessly without any third-party programs. Package manifests are periodically fetched from the GitHub API to prevent hitting ratelimits. 4 | 5 | If you wish to use our API, please take a look at [our docs](https://docs.winget.run). All other non-documentation info will be provided in this readme. 6 | 7 | ## Contents 8 | - [Installation](#installation) 9 | - [Development](#development) 10 | - [Deployment](#deployment) 11 | - [Contributing](#contributing) 12 | - [Authors](#authors) 13 | - [Acknowledgments](#acknowledgments) 14 | - [License](#license) 15 | 16 | ## Installation 17 | 18 | A Docker image is built for the project in our CI/CD pipeline on the develop, release/* and master branches. These can be found [here](https://github.com/winget-run/api/packages/236685). A detailed example of building and running the project without docker can be found in the [development](#Development) section. 19 | 20 | > NOTE: We currently only support x509 MongoDB autnetication with TLS, we may modify this at a later date. 21 | 22 | The following environment variabled are required to run the container: 23 | - **MONGO_HOST**: MongoDB host. 24 | - **MONGO_DB**: MongoDB database name. 25 | - **MONGO_CERT**: MongoDB x509 cert. 26 | - **MONGO_CA**: MongoDB CA cert. 27 | - **WEB_ADDRESS**: Host address for CORS. 28 | - **WEBSERVER_LOGGER**: Enable logger (boolean). 29 | - **WEBSERVER_PORT**: Port to run the API on. 30 | - **WEBSERVER_ADDRESS**: Address to run the server on (eg. 0.0.0.0). 31 | - **GITHUB_TOKEN**: GitHub API token. 32 | - **CRON_FREQUENCY**: Cron notation for UPDATE_FREQUENCY_MINUTES (below). 33 | - **UPDATE_FREQUENCY_MINUTES**: How often new packages are fetched from GitHub in minutes. 34 | - **API_ACCESS_TOKEN**: Token that will be required for accessing protected routes. 35 | 36 | > NOTE: The cron job is not included in this app and needs to be set up seperately. 37 | 38 | ## Development 39 | 40 | Local development requires the following software: 41 | - NodeJS 42 | - Yarn 43 | - MongoDB 44 | 45 | The environment variables mentioned in the [installation](#Installation) section can be placed in a .env file in the project's root. 46 | 47 | If everything is set up correctly, run the following command for an optimal development environment, which will watch for changes in the typescript files and auto-restart the server if necessary. 48 | - `yarn build:watch` 49 | - `yarn run:hot` 50 | 51 | Tests and linting can be run using the following commands: 52 | - `yarn test` 53 | - `yarn lint` 54 | 55 | For any additional commands, check out the package.json. 56 | 57 | > NOTE: The MongoDB ORM that were using, typeorm, currently doesn't support TLS, making it work unfortunately required a monkey patch in /src/database/index.ts. 58 | 59 | ## Deployment 60 | 61 | We use GitHub Actions CI/CD and Kubernetes for our deployments. All required into regarding deployments can be found in /.github and /chart. 62 | 63 | ## Contributing 64 | 65 | Issues and pull requests are welcome. We currently don't have any templates (at the time of writing) so a pr for those would be nice as well. If you wish to check the progress of current tickets, we have boards set up using [ZenHub](https://www.zenhub.com/). 66 | 67 | We currently don't have tests, but will add them soon™. 68 | 69 | ## Authors 70 | 71 | - **Lukasz Niezabitowski** *(Dragon1320)* 72 | - **Ryan Larkin** *(rlarkin212)* 73 | - **Matthew Watt** *(MattheousDT)* 74 | 75 | ## Acknowledgments 76 | 77 | - My beloved coffee machine for making glorious coffee in the morning (and night) and keeping me awake during these 12 hour programming sessions as we rushed to get this released. 78 | - Certain things mentioned in our docs' introduction section and certain other things that I was not allowed to leave in but kept in source control anyway to amuse anyone who comes across it. 79 | 80 | ## License 81 | 82 | Ask us if you want to use the code, or suggest a license and make a pr. 83 | -------------------------------------------------------------------------------- /chart/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /chart/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: api 3 | description: winget.run rest api 4 | 5 | type: application 6 | 7 | # chart version 8 | version: 0.1.0 9 | # app version 10 | appVersion: 0.1.0 11 | -------------------------------------------------------------------------------- /chart/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: {{ .Release.Name }} 5 | data: 6 | WEB_ADDRESS: "*" 7 | WEBSERVER_LOGGER: "true" 8 | WEBSERVER_PORT: "3000" 9 | WEBSERVER_ADDRESS: "0.0.0.0" 10 | 11 | MONGO_HOST: mongo-main-0.mongo-service.mongodb.svc.cluster.local,mongo-main-1.mongo-service.mongodb.svc.cluster.local 12 | MONGO_DB: winget-run-dev 13 | -------------------------------------------------------------------------------- /chart/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ .Release.Name }} 5 | spec: 6 | replicas: 1 7 | strategy: 8 | type: RollingUpdate 9 | rollingUpdate: 10 | maxUnavailable: 50% 11 | maxSurge: 1 12 | selector: 13 | matchLabels: 14 | app: {{ .Release.Name }} 15 | template: 16 | metadata: 17 | labels: 18 | app: {{ .Release.Name }} 19 | annotations: 20 | # set automatically by github actions 21 | sha: {{ .Values.sha }} 22 | spec: 23 | imagePullSecrets: 24 | - name: regcred 25 | containers: 26 | - name: {{ .Release.Name }} 27 | image: docker.pkg.github.com/winget-run/{{ .Release.Name }}/{{ .Release.Name }}:develop 28 | imagePullPolicy: Always 29 | ports: 30 | - containerPort: 3000 31 | readinessProbe: 32 | httpGet: 33 | path: /healthz 34 | port: 3000 35 | initialDelaySeconds: 5 36 | periodSeconds: 5 37 | successThreshold: 1 38 | env: 39 | - name: MONGO_CERT 40 | valueFrom: 41 | secretKeyRef: 42 | key: MONGO_CERT 43 | name: {{ .Release.Name }} 44 | - name: GITHUB_TOKEN 45 | valueFrom: 46 | secretKeyRef: 47 | key: GITHUB_TOKEN 48 | name: {{ .Release.Name }} 49 | - name: API_ACCESS_TOKEN 50 | valueFrom: 51 | secretKeyRef: 52 | key: API_ACCESS_TOKEN 53 | name: {{ .Release.Name }} 54 | 55 | - name: WEB_ADDRESS 56 | valueFrom: 57 | configMapKeyRef: 58 | key: WEB_ADDRESS 59 | name: {{ .Release.Name }} 60 | - name: WEBSERVER_LOGGER 61 | valueFrom: 62 | configMapKeyRef: 63 | key: WEBSERVER_LOGGER 64 | name: {{ .Release.Name }} 65 | - name: WEBSERVER_PORT 66 | valueFrom: 67 | configMapKeyRef: 68 | key: WEBSERVER_PORT 69 | name: {{ .Release.Name }} 70 | - name: WEBSERVER_ADDRESS 71 | valueFrom: 72 | configMapKeyRef: 73 | key: WEBSERVER_ADDRESS 74 | name: {{ .Release.Name }} 75 | 76 | - name: MONGO_HOST 77 | valueFrom: 78 | configMapKeyRef: 79 | key: MONGO_HOST 80 | name: {{ .Release.Name }} 81 | - name: MONGO_DB 82 | valueFrom: 83 | configMapKeyRef: 84 | key: MONGO_DB 85 | name: {{ .Release.Name }} 86 | -------------------------------------------------------------------------------- /chart/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ .Release.Name }} 5 | spec: 6 | type: ClusterIP 7 | ports: 8 | - port: 80 9 | targetPort: 3000 10 | selector: 11 | app: {{ .Release.Name }} 12 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | roots: [ 3 | "/tests", 4 | "/src", 5 | ], 6 | transform: { 7 | "^.+\\.ts?$": "ts-jest", 8 | }, 9 | coverageThreshold: { 10 | global: { 11 | branches: 50, 12 | functions: 50, 13 | lines: 50, 14 | statements: 50, 15 | }, 16 | }, 17 | coverageReporters: [ 18 | "json", 19 | "lcov", 20 | "text", 21 | "clover", 22 | ], 23 | setupFiles: [ 24 | "/tests/setup.ts", 25 | ], 26 | }; 27 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "winget-api", 3 | "version": "0.2.0", 4 | "description": "view winget packages", 5 | "author": { 6 | "name": "Feinwaru", 7 | "email": "feinwaru@gmail.com", 8 | "url": "https://feinwaru.com" 9 | }, 10 | "license": "UNLICENSED", 11 | "private": true, 12 | "scripts": { 13 | "lint": "yarn eslint . --ext .ts", 14 | "lint:fix": "yarn eslint . --ext .ts --fix", 15 | "test": "yarn cross-env NODE_ENV=test NODE_PATH=. yarn jest --coverage", 16 | "test:watch": "yarn cross-env NODE_ENV=test NODE_PATH=. yarn jest --coverage --watchAll", 17 | "view:coverage": "yarn serve coverage/lcov-report", 18 | "build": "yarn tsc", 19 | "build:watch": "yarn tsc --watch", 20 | "run:dev": "yarn cross-env NODE_ENV=dev NODE_PATH=. node -r dotenv/config -r source-map-support/register dist | yarn pino-pretty -c -l", 21 | "run:hot": "yarn cross-env NODE_ENV=dev NODE_PATH=. yarn nodemon -r dotenv/config -r source-map-support/register dist --watch dist --watch .env | yarn pino-pretty -c -l", 22 | "run:prod": "yarn cross-env NODE_ENV=prod NODE_PATH=. node dist", 23 | "kube:dev": "telepresence --swap-deployment api --namespace winget-run-dev --expose 3000 --method inject-tcp --run yarn cross-env NODE_ENV=dev NODE_PATH=. node -r dotenv/config -r source-map-support/register dist | yarn pino-pretty -c -l", 24 | "kube:hot": "telepresence --swap-deployment api --namespace winget-run-dev --expose 3000 --method inject-tcp --run yarn cross-env NODE_ENV=dev NODE_PATH=. yarn nodemon -r dotenv/config -r source-map-support/register dist --watch dist --watch .env | yarn pino-pretty -c -l", 25 | "kube:cunt": "telepresence --swap-deployment api --namespace winget-run-dev --expose 3000 --run yarn cross-env NODE_ENV=dev NODE_PATH=. yarn nodemon -r dotenv/config -r source-map-support/register dist --watch dist --watch .env | yarn pino-pretty -c -l", 26 | "clean": "rm -rf dist coverage", 27 | 28 | "format:check": "echo format:check", 29 | "lint:check": "echo lint:check", 30 | "test:check": "echo test:check", 31 | "build:check": "echo build:check" 32 | }, 33 | "dependencies": { 34 | "@types/encoding-japanese": "^1.0.15", 35 | "@types/lodash": "^4.14.153", 36 | "@types/moment": "^2.13.0", 37 | "bson": "^4.0.4", 38 | "cross-env": "^7.0.2", 39 | "dotenv": "^8.2.0", 40 | "encoding-japanese": "^1.0.30", 41 | "fastify": "^2.15.1", 42 | "fastify-cors": "^3.0.3", 43 | "js-yaml": "^3.14.0", 44 | "moment": "^2.27.0", 45 | "mongodb": "^3.5.7", 46 | "natural": "^2.1.5", 47 | "node-cache": "^5.1.2", 48 | "node-fetch": "^2.6.0", 49 | "reflect-metadata": "^0.1.13", 50 | "typeorm": "^0.2.25", 51 | "uuid-mongodb": "^2.3.0" 52 | }, 53 | "devDependencies": { 54 | "@types/bson": "^4.0.2", 55 | "@types/dotenv": "^8.2.0", 56 | "@types/jest": "^25.2.3", 57 | "@types/js-yaml": "^3.12.4", 58 | "@types/mongodb": "^3.5.18", 59 | "@types/natural": "^0.6.3", 60 | "@types/node": "^14.0.5", 61 | "@types/node-cache": "^4.2.5", 62 | "@types/node-fetch": "^2.5.7", 63 | "@types/source-map-support": "^0.5.1", 64 | "@typescript-eslint/eslint-plugin": "^2.34.0", 65 | "@typescript-eslint/parser": "^2.34.0", 66 | "eslint": "^6.0.0", 67 | "eslint-config-airbnb-base": "^14.1.0", 68 | "eslint-plugin-import": "^2.20.2", 69 | "jest": "^26.0.1", 70 | "nodemon": "^2.0.4", 71 | "pino-pretty": "^4.1.0", 72 | "serve": "^11.3.0", 73 | "source-map-support": "^0.5.19", 74 | "ts-jest": "^26.0.0", 75 | "typescript": "^3.9.3" 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/database/helpers/index.ts: -------------------------------------------------------------------------------- 1 | import muuid from "uuid-mongodb"; 2 | // import rfdc from "rfdc"; 3 | 4 | import { 5 | padSemver, 6 | sortSemver, 7 | rebuildPackage, 8 | addOrUpdatePackage, 9 | removePackage, 10 | generateMetaphones, 11 | generateNGrams, 12 | } from "./package"; 13 | import { 14 | IBase, 15 | IBaseFindManyOptions, 16 | IBaseInternalFindManyOptions, 17 | IBaseFindOneOptions, 18 | IBaseInternalFindOneOptions, 19 | IBaseFilters, 20 | IBaseInternalFilters, 21 | } from "../types"; 22 | 23 | // const clone = rfdc(); 24 | 25 | // TODO: refactor these into 1 or 2, somehow... 26 | const mapInternalFilters = (filters: IBaseFilters): IBaseInternalFilters => { 27 | // const clonedFilters = clone(filters); 28 | 29 | const mappedFilters = { 30 | ...filters, 31 | ...(filters.uuid == null ? {} : { _id: muuid.from(filters.uuid) }), 32 | uuid: undefined, 33 | }; 34 | 35 | // Reflect.deleteProperty(mappedFilters, "uuid"); 36 | 37 | return mappedFilters; 38 | }; 39 | 40 | const mapInternalFindOneOptions = (options: IBaseFindOneOptions): IBaseInternalFindOneOptions => { 41 | // const clonedOptions = clone(options); 42 | 43 | const mappedOptions = { 44 | ...options, 45 | filters: undefined, 46 | 47 | where: { 48 | ...options.filters, 49 | ...(options.filters?.uuid == null ? {} : { _id: muuid.from(options.filters.uuid) }), 50 | 51 | uuid: undefined, 52 | }, 53 | }; 54 | 55 | // Reflect.deleteProperty(mappedOptions, "filters"); 56 | // Reflect.deleteProperty(mappedOptions.where, "uuid"); 57 | 58 | return mappedOptions; 59 | }; 60 | 61 | const mapInternalFindManyOptions = (options: IBaseFindManyOptions): IBaseInternalFindManyOptions => { 62 | // const clonedOptions = clone(options); 63 | 64 | // console.log(options); 65 | 66 | const mappedOptions = { 67 | ...options, 68 | filters: undefined, 69 | 70 | where: { 71 | ...options.filters, 72 | ...(options.filters?.uuid == null ? {} : { _id: muuid.from(options.filters.uuid) }), 73 | uuid: undefined, 74 | }, 75 | }; 76 | 77 | // Reflect.deleteProperty(mappedOptions, "filters"); 78 | // Reflect.deleteProperty(mappedOptions.where, "uuid"); 79 | 80 | return mappedOptions; 81 | }; 82 | 83 | // NOTE: can make this more complex to work with more data types but this will do for now 84 | const dedupe = (array: string[]): string[] => array.filter((e, i, a) => i === a.findIndex(f => e === f)); 85 | 86 | // mdn: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions 87 | const escapeRegex = (str: string): string => str.replace(/[.*+\-?^${}()|[\]\\]/g, "\\$&"); 88 | 89 | export { 90 | mapInternalFilters, 91 | mapInternalFindOneOptions, 92 | mapInternalFindManyOptions, 93 | 94 | padSemver, 95 | sortSemver, 96 | rebuildPackage, 97 | addOrUpdatePackage, 98 | removePackage, 99 | generateMetaphones, 100 | generateNGrams, 101 | 102 | dedupe, 103 | escapeRegex, 104 | }; 105 | -------------------------------------------------------------------------------- /src/database/helpers/package.ts: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | import path from "path"; 3 | 4 | import { DoubleMetaphone, NGrams, TfIdf } from "natural"; 5 | 6 | import NodeCache from "node-cache"; 7 | 8 | import { ManifestService, PackageService } from "../service"; 9 | import { 10 | IManifest, 11 | IBaseInsert, 12 | IBaseUpdate, 13 | IPackage, 14 | } from "../types"; 15 | 16 | const { 17 | NODE_PATH, 18 | } = process.env; 19 | 20 | const NGRAM_MIN = 2; 21 | 22 | const TFIDF_SAMPLE_CACHE_SEC = 30; 23 | const TFIDF_SAMPLE_NAME = "desc_sample"; 24 | const TFIDF_SAMPLE_FP = path.join(NODE_PATH, "assets", `${TFIDF_SAMPLE_NAME}.json`); 25 | 26 | const cache = new NodeCache(); 27 | 28 | // ye theres gonna be longer versions with random characters but those are technically 29 | // against spec and should break anything sooooo... 30 | const padSemver = (version: string): string => version.split(".").map(e => e.padStart(5, "0")).join("."); 31 | 32 | enum SortDirection { 33 | Ascending = 1, 34 | Descending = -1, 35 | } 36 | 37 | const createSortSemver = (direction: SortDirection) => (a: string, b: string): number => { 38 | const aPad = padSemver(a); 39 | const bPad = padSemver(b); 40 | 41 | if (aPad > bPad) { 42 | return direction; 43 | } 44 | 45 | if (aPad < bPad) { 46 | return -direction; 47 | } 48 | 49 | return 0; 50 | }; 51 | 52 | // TODO: remove (dead code?) 53 | const sortSemver = (a: string, b: string): number => { 54 | const aPad = padSemver(a); 55 | const bPad = padSemver(b); 56 | 57 | if (aPad > bPad) { 58 | return 1; 59 | } 60 | if (aPad < bPad) { 61 | return -1; 62 | } 63 | 64 | return 0; 65 | }; 66 | 67 | const generateMetaphones = (word: string): string[] => { 68 | const metaphones = DoubleMetaphone.process(word); 69 | if (metaphones[0] === metaphones[1]) { 70 | return metaphones.slice(1); 71 | } 72 | 73 | return metaphones; 74 | }; 75 | 76 | const generateNGrams = (word: string, min: number): string[] => { 77 | const ngrams = []; 78 | 79 | const encodings = generateMetaphones(word); 80 | ngrams.push([encodings]); 81 | 82 | for (let i = 0; i < encodings.length; i += 1) { 83 | if (encodings[i].length === Math.max(1, min)) { 84 | // TODO: if both encodings will always be the same length, remove this line 85 | // see this would be redundant cos im already adding the encodings a couple lines up 86 | // BUT idk if one encoding can be longer than other so ill leave it like this for now 87 | ngrams.push([[encodings[i]]]); 88 | } else { 89 | for (let j = min; j < encodings[i].length; j += 1) { 90 | ngrams.push(NGrams.ngrams(encodings[i].split(""), j)); 91 | } 92 | } 93 | } 94 | 95 | return ngrams.flat().map(e => e.reduce((a, c) => a + c, "")).filter((e, i, a) => i === a.findIndex(f => e === f)).map(e => e.padEnd(3, "_")); 96 | }; 97 | 98 | const generateStartNGrams = (word: string, min: number): string[] => { 99 | const ngrams = []; 100 | 101 | const encodings = generateMetaphones(word); 102 | 103 | for (let i = 0; i < encodings.length; i += 1) { 104 | if (encodings[i].length === Math.max(1, min)) { 105 | // TODO: if both encodings will always be the same length, remove this line 106 | // see this would be redundant cos im already adding the encodings a couple lines up 107 | // BUT idk if one encoding can be longer than other so ill leave it like this for now 108 | ngrams.push(encodings[i]); 109 | } else { 110 | for (let j = min; j <= encodings[i].length; j += 1) { 111 | ngrams.push(encodings[i].slice(0, j)); 112 | } 113 | } 114 | } 115 | 116 | return ngrams.filter((e, i, a) => i === a.findIndex(f => e === f)).map(e => e.padEnd(3, "_")); 117 | }; 118 | 119 | const extractKeywords = (text: string, max?: number): string[] => { 120 | let tfidf: TfIdf; 121 | 122 | const cachedTfidfData: string | undefined = cache.get(TFIDF_SAMPLE_NAME); 123 | if (cachedTfidfData == null) { 124 | const descSample: string[] = JSON.parse(fs.readFileSync(TFIDF_SAMPLE_FP).toString()); 125 | 126 | tfidf = new TfIdf(); 127 | descSample.forEach(e => tfidf.addDocument(e)); 128 | 129 | cache.set(TFIDF_SAMPLE_NAME, JSON.stringify(tfidf), TFIDF_SAMPLE_CACHE_SEC); 130 | } else { 131 | tfidf = new TfIdf(JSON.parse(cachedTfidfData)); 132 | } 133 | 134 | tfidf.addDocument(text); 135 | // it does exist, the types are just fucked as usual 136 | // TODO: can probs set to unknown and run some validation on this 137 | const keywords = tfidf.listTerms((tfidf as any).documents.length - 1); 138 | 139 | return keywords.slice(0, max ?? keywords.length).map(e => e.term); 140 | }; 141 | 142 | // NOTE: pkg are any additional fields that should be updated or overwritten on the package doc 143 | const rebuildPackage = async (id: string, pkg: IBaseUpdate = {}): Promise => { 144 | const manifestService = new ManifestService(); 145 | const packageService = new PackageService(); 146 | 147 | // TODO: optimisation, dont fetch every manifest for a package every 148 | // single time (can get unweildy when more versions are added) 149 | const manifests = await manifestService.find({ 150 | filters: { 151 | Id: id, 152 | }, 153 | }); 154 | 155 | if (manifests.length === 0) { 156 | await packageService.deleteOne({ 157 | Id: id, 158 | }); 159 | 160 | return; 161 | } 162 | 163 | // TODO: just sort the manifests array instead 164 | 165 | // get fields from latest 166 | // get version list 167 | // get sortable version field 168 | const versions = manifests.map(e => e.Version).sort(createSortSemver(SortDirection.Descending)); 169 | const latestVersion = versions[0]; 170 | // doing a manifests.length check a few lines up 171 | const latestManifest = manifests.find(e => e.Version === latestVersion) as IManifest; 172 | // 173 | 174 | const tags = latestManifest.Tags == null ? [] : latestManifest.Tags.split(",").map(e => e.trim().toLowerCase()); 175 | 176 | // search shite 177 | const tagNGrams = tags.map(e => generateNGrams(e, NGRAM_MIN)).flat().filter((e, i, a) => i === a.findIndex(f => e === f)); 178 | 179 | // optimisations: 180 | // - remove short words 181 | // - only make start of word ngrams 182 | // - ...or dont make ngrams at all? 183 | // - set max description words/length? 184 | // - some way of picking out key words 185 | 186 | // TODO: also adjust field weights again 187 | // const descriptionNGrams = latestManifest.Description == null ? [] : generateNGrams(latestManifest.Description, NGRAM_MIN); 188 | 189 | const descriptionNGrams = latestManifest.Description == null 190 | ? [] 191 | : extractKeywords(latestManifest.Description, 10) 192 | .map(e => generateNGrams(e, 2)) 193 | .flat() 194 | .filter((e, i, a) => i === a.findIndex(f => e === f)); 195 | 196 | const newPkg = { 197 | Id: latestManifest.Id, 198 | 199 | Versions: versions, 200 | Latest: { 201 | Name: latestManifest.Name, 202 | Publisher: latestManifest.Publisher, 203 | Tags: tags, 204 | Description: latestManifest.Description, 205 | Homepage: latestManifest.Homepage, 206 | License: latestManifest.License, 207 | LicenseUrl: latestManifest.LicenseUrl, 208 | }, 209 | 210 | // Featured: false, 211 | 212 | Search: { 213 | Name: generateNGrams(latestManifest.Name, NGRAM_MIN).join(" "), 214 | Publisher: generateNGrams(latestManifest.Publisher, NGRAM_MIN).join(" "), 215 | Tags: tagNGrams.length === 0 ? undefined : tagNGrams.join(" "), 216 | Description: descriptionNGrams.length === 0 ? undefined : descriptionNGrams.join(" "), 217 | }, 218 | 219 | UpdatedAt: new Date(), 220 | CreatedAt: new Date(), 221 | 222 | ...pkg, 223 | }; 224 | 225 | packageService.repository.updateOne( 226 | { 227 | Id: id, 228 | }, 229 | { 230 | $set: newPkg, 231 | $setOnInsert: { 232 | Featured: false, 233 | }, 234 | }, 235 | { 236 | upsert: true, 237 | }, 238 | ); 239 | }; 240 | 241 | // NOTE: additions are made synchronously as data integrity is more important than speed here 242 | 243 | // NOTE: for simplicity, a package is completely re-added/updated after a corresponding manifest change 244 | // this 1. isnt a big issue cos reads >>> writes, and 2. manifest errors can be easily fixed 245 | const addOrUpdatePackage = async (manifest: IBaseInsert, pkg: IBaseUpdate = {}): Promise => { 246 | const manifestService = new ManifestService(); 247 | 248 | const { Id: id } = manifest; 249 | 250 | if (id == null) { 251 | throw new Error("id not set"); 252 | } 253 | 254 | await manifestService.upsertManifest(manifest); 255 | await rebuildPackage(id, pkg); 256 | }; 257 | 258 | const removePackage = async (id: string, version: string, pkg: IBaseUpdate = {}): Promise => { 259 | const manifestService = new ManifestService(); 260 | 261 | await manifestService.removeManifestVersion(id, version); 262 | await rebuildPackage(id, pkg); 263 | }; 264 | 265 | export { 266 | padSemver, 267 | sortSemver, 268 | rebuildPackage, 269 | addOrUpdatePackage, 270 | removePackage, 271 | generateNGrams, 272 | generateStartNGrams, 273 | generateMetaphones, 274 | extractKeywords, 275 | }; 276 | -------------------------------------------------------------------------------- /src/database/index.ts: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | import path from "path"; 3 | 4 | import { createConnection } from "typeorm"; 5 | 6 | import { PackageModel, ManifestModel, StatsModel } from "./model"; 7 | import { PackageService, ManifestService, StatsService } from "./service"; 8 | import { 9 | IPackage, 10 | IManifest, 11 | StatsResolution, 12 | IStats, 13 | } from "./types"; 14 | import { 15 | padSemver, 16 | sortSemver, 17 | rebuildPackage, 18 | addOrUpdatePackage, 19 | removePackage, 20 | } from "./helpers"; 21 | 22 | // patch the typeorm MongoDriver to support mongo 3.6+ tls options 23 | // eslint-disable-next-line @typescript-eslint/no-var-requires 24 | const MongoDriver = require("typeorm/driver/mongodb/MongoDriver"); 25 | 26 | const originalFunc = MongoDriver.MongoDriver.prototype.buildConnectionOptions; 27 | // eslint-disable-next-line func-names 28 | MongoDriver.MongoDriver.prototype.buildConnectionOptions = function (): object { 29 | const mongoOptions = { 30 | ...originalFunc.apply(this), 31 | ...this.options.extra, 32 | }; 33 | 34 | return mongoOptions; 35 | }; 36 | 37 | const CA_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"; 38 | 39 | const { 40 | NODE_ENV, 41 | NODE_PATH, 42 | MONGO_HOST, 43 | MONGO_DB, 44 | MONGO_CERT, 45 | MONGO_CA, 46 | } = process.env; 47 | 48 | const connect = async (): Promise => { 49 | const certPath = path.join(NODE_PATH, "mongo-cert.pem"); 50 | fs.writeFileSync(certPath, MONGO_CERT); 51 | 52 | const caPath = path.join(NODE_PATH, "mongo-ca.crt"); 53 | if (MONGO_CA != null) { 54 | fs.writeFileSync(caPath, MONGO_CA); 55 | } 56 | 57 | const envOptions = { 58 | ...(NODE_ENV === "dev" ? { tlsCAFile: caPath } : {}), 59 | ...(NODE_ENV === "prod" ? { tlsCAFile: MONGO_CA == null ? CA_PATH : caPath } : {}), 60 | }; 61 | 62 | await createConnection({ 63 | type: "mongodb", 64 | // needs to be set in the url, if we try to set the 'database' field, it tries to connect to a 'test' db... 65 | url: `mongodb://${MONGO_HOST}/${MONGO_DB}`, 66 | useNewUrlParser: true, 67 | useUnifiedTopology: true, 68 | authMechanism: "MONGODB-X509", 69 | authSource: "$external", 70 | 71 | // requires a patch to work with mongo 3.6+ features (tls) 72 | extra: { 73 | tls: true, 74 | tlsCertificateKeyFile: certPath, 75 | 76 | ...envOptions, 77 | }, 78 | 79 | entities: [ 80 | PackageModel, 81 | ManifestModel, 82 | StatsModel, 83 | ], 84 | }); 85 | 86 | // ensure appropriate indexes (since the typeorm way to do it is broke) 87 | const manifestService = new ManifestService(); 88 | manifestService.setupIndices(); 89 | 90 | const packageService = new PackageService(); 91 | packageService.setupIndices(); 92 | 93 | console.log(`connected to mongo; ${MONGO_HOST}/${MONGO_DB}`); 94 | }; 95 | 96 | export { 97 | connect, 98 | 99 | PackageModel, 100 | IPackage, 101 | PackageService, 102 | 103 | ManifestModel, 104 | IManifest, 105 | ManifestService, 106 | 107 | StatsModel, 108 | StatsResolution, 109 | IStats, 110 | StatsService, 111 | 112 | padSemver, 113 | sortSemver, 114 | rebuildPackage, 115 | addOrUpdatePackage, 116 | removePackage, 117 | }; 118 | -------------------------------------------------------------------------------- /src/database/model/base.ts: -------------------------------------------------------------------------------- 1 | import muuid from "uuid-mongodb"; 2 | import { Binary } from "bson"; 3 | import { Column, PrimaryColumn } from "typeorm"; 4 | 5 | import { IBase } from "../types/base"; 6 | 7 | class BaseModel implements IBase { 8 | @PrimaryColumn() 9 | _id!: object; 10 | 11 | @Column() 12 | __v!: number; 13 | 14 | @Column() 15 | createdAt!: Date; 16 | 17 | @Column() 18 | updatedAt!: Date; 19 | 20 | // virtuals 21 | public get uuid(): string { 22 | return muuid.from(this._id as Binary).toString(); 23 | } 24 | } 25 | 26 | export default BaseModel; 27 | -------------------------------------------------------------------------------- /src/database/model/index.ts: -------------------------------------------------------------------------------- 1 | import BaseModel from "./base"; 2 | import PackageModel from "./package"; 3 | import ManifestModel from "./manifest"; 4 | import StatsModel from "./stats"; 5 | 6 | export { 7 | BaseModel, 8 | PackageModel, 9 | ManifestModel, 10 | StatsModel, 11 | }; 12 | -------------------------------------------------------------------------------- /src/database/model/manifest.ts: -------------------------------------------------------------------------------- 1 | import { Column, Entity } from "typeorm"; 2 | 3 | import BaseModel from "./base"; 4 | import { IManifest } from "../types"; 5 | 6 | @Entity() 7 | class ManifestModel extends BaseModel implements IManifest { 8 | @Column() 9 | public Id!: string; 10 | 11 | @Column() 12 | public Name!: string; 13 | 14 | @Column() 15 | public AppMoniker?: string; 16 | 17 | @Column() 18 | public Version!: string; 19 | 20 | @Column() 21 | public Publisher!: string; 22 | 23 | @Column() 24 | public Channel?: string; 25 | 26 | @Column() 27 | public Author?: string; 28 | 29 | @Column() 30 | public License?: string; 31 | 32 | @Column() 33 | public LicenseUrl?: string; 34 | 35 | @Column() 36 | public MinOSVersion?: string; 37 | 38 | @Column() 39 | public Description?: string; 40 | 41 | @Column() 42 | public Homepage?: string; 43 | 44 | @Column() 45 | public Tags?: string; 46 | 47 | @Column() 48 | public FileExtensions?: string; 49 | 50 | @Column() 51 | public Protocols?: string; 52 | 53 | @Column() 54 | public Commands?: string; 55 | 56 | @Column() 57 | public InstallerType?: string; 58 | 59 | @Column() 60 | public Switches?: { 61 | Custom?: string; 62 | Silent?: string; 63 | SilentWithProgress?: string; 64 | Interactive?: string; 65 | Language?: string; 66 | }; 67 | 68 | @Column() 69 | public Log?: string; 70 | 71 | @Column() 72 | public InstallLocation?: string; 73 | 74 | @Column() 75 | public Installers!: [ 76 | { 77 | Arch: string; 78 | Url: string; 79 | Sha256: string; 80 | SignatureSha256?: string; 81 | Language?: string; 82 | InstallerType?: string; 83 | Scope?: string; 84 | SystemAppId?: string; 85 | Switches?: { 86 | Language?: string; 87 | Custom?: string; 88 | }; 89 | } 90 | ]; 91 | 92 | @Column() 93 | public Localization?: [ 94 | { 95 | Language: string; 96 | Description?: string; 97 | Homepage?: string; 98 | LicenseUrl?: string; 99 | } 100 | ]; 101 | } 102 | 103 | export default ManifestModel; 104 | -------------------------------------------------------------------------------- /src/database/model/package.ts: -------------------------------------------------------------------------------- 1 | import { Column, Entity } from "typeorm"; 2 | 3 | import BaseModel from "./base"; 4 | import { IPackage } from "../types/package"; 5 | 6 | @Entity() 7 | class PackageModel extends BaseModel implements IPackage { 8 | @Column() 9 | Id!: string; 10 | 11 | // version stuff 12 | @Column() 13 | Versions!: string[]; 14 | 15 | @Column() 16 | Latest!: { 17 | Name: string; 18 | Publisher: string; 19 | Tags: string[]; 20 | Description?: string; 21 | Homepage?: string; 22 | License?: string; 23 | LicenseUrl?: string; 24 | }; 25 | 26 | // extra 27 | @Column() 28 | Featured!: boolean; 29 | 30 | @Column() 31 | IconUrl?: string; 32 | 33 | @Column() 34 | Banner?: string; 35 | 36 | @Column() 37 | Logo?: string; 38 | 39 | // search 40 | @Column() 41 | Search!: { 42 | Name: string; 43 | Publisher: string; 44 | Tags?: string; 45 | Description?: string; 46 | }; 47 | 48 | // stats 49 | @Column() 50 | UpdatedAt!: Date; 51 | 52 | @Column() 53 | CreatedAt!: Date; 54 | } 55 | 56 | export default PackageModel; 57 | -------------------------------------------------------------------------------- /src/database/model/stats.ts: -------------------------------------------------------------------------------- 1 | import { Column, Entity } from "typeorm"; 2 | 3 | import BaseModel from "./base"; 4 | import { IStats } from "../types"; 5 | 6 | @Entity() 7 | class StatsModel extends BaseModel implements IStats { 8 | @Column() 9 | public PackageId!: string; 10 | 11 | @Column() 12 | public Period!: number; 13 | 14 | @Column() 15 | public Value!: number; 16 | } 17 | 18 | export default StatsModel; 19 | -------------------------------------------------------------------------------- /src/database/service/base.ts: -------------------------------------------------------------------------------- 1 | import { MongoRepository } from "typeorm"; 2 | import muuid from "uuid-mongodb"; 3 | 4 | import BaseModel from "../model/base"; 5 | import { mapInternalFindManyOptions, mapInternalFindOneOptions, mapInternalFilters } from "../helpers"; 6 | import { 7 | IBaseFindManyOptions, 8 | IBaseFindOneOptions, 9 | IBaseUpdateOptions, 10 | IBaseUpdate, 11 | IBaseFilters, 12 | IUpdateResult, 13 | IDeleteResult, 14 | IBaseInsert, 15 | IInsertResult, 16 | } from "../types"; 17 | 18 | // TODO: handle errors properly 19 | // TODO: figure out a way to return uuid instead of _id (mongoose equivalent of toObject or whatever) 20 | abstract class BaseService { 21 | protected repository!: MongoRepository 22 | 23 | public async find(options: IBaseFindManyOptions): Promise { 24 | try { 25 | const internalOptions = mapInternalFindManyOptions(options); 26 | 27 | return this.repository.find(internalOptions); 28 | } catch (error) { 29 | throw new Error(error); 30 | } 31 | } 32 | 33 | public async count(options: IBaseFindManyOptions): Promise { 34 | try { 35 | const internalOptions = mapInternalFindManyOptions(options); 36 | 37 | return this.repository.count(internalOptions); 38 | } catch (error) { 39 | throw new Error(error); 40 | } 41 | } 42 | 43 | public async findAndCount(options: IBaseFindManyOptions): Promise<[T[], number]> { 44 | try { 45 | const internalOptions = mapInternalFindManyOptions(options); 46 | 47 | return this.repository.findAndCount(internalOptions); 48 | } catch (error) { 49 | throw new Error(error); 50 | } 51 | } 52 | 53 | public async findOne(options: IBaseFindOneOptions): Promise { 54 | try { 55 | const internalOptions = mapInternalFindOneOptions(options); 56 | 57 | return this.repository.findOne(internalOptions); 58 | } catch (error) { 59 | throw new Error(error); 60 | } 61 | } 62 | 63 | public async findOneById(uuid: string): Promise { 64 | try { 65 | const internalOptions = mapInternalFindOneOptions({ filters: { uuid } }); 66 | 67 | return this.repository.findOne(internalOptions); 68 | } catch (error) { 69 | throw new Error(error); 70 | } 71 | } 72 | 73 | public async insertOne(insert: IBaseInsert): Promise { 74 | try { 75 | return this.repository.insertOne({ 76 | ...insert, 77 | 78 | _id: muuid.v4(), 79 | createdAt: new Date(), 80 | updatedAt: new Date(), 81 | }); 82 | } catch (error) { 83 | throw new Error(error); 84 | } 85 | } 86 | 87 | public async update(options: IBaseUpdateOptions): Promise { 88 | try { 89 | const internalFilters = mapInternalFilters(options.filters ?? {}); 90 | 91 | return this.repository.updateMany(internalFilters, { 92 | $set: { 93 | ...options.update, 94 | 95 | updatedAt: new Date(), 96 | }, 97 | }); 98 | } catch (error) { 99 | throw new Error(error); 100 | } 101 | } 102 | 103 | public async updateOne(options: IBaseUpdateOptions): Promise { 104 | try { 105 | const internalFilters = mapInternalFilters(options.filters ?? {}); 106 | 107 | return this.repository.updateOne(internalFilters, { 108 | $set: { 109 | ...options.update, 110 | 111 | updatedAt: new Date(), 112 | }, 113 | }); 114 | } catch (error) { 115 | throw new Error(error); 116 | } 117 | } 118 | 119 | public async updateOneById(uuid: string, update: IBaseUpdate): Promise { 120 | try { 121 | const internalFilters = mapInternalFilters({ uuid }); 122 | 123 | return this.repository.updateOne(internalFilters, { 124 | $set: { 125 | ...update, 126 | 127 | updatedAt: new Date(), 128 | }, 129 | }); 130 | } catch (error) { 131 | throw new Error(error); 132 | } 133 | } 134 | 135 | public async delete(filters: IBaseFilters): Promise { 136 | try { 137 | const internalFilters = mapInternalFilters(filters); 138 | 139 | return this.repository.deleteMany(internalFilters); 140 | } catch (error) { 141 | throw new Error(error); 142 | } 143 | } 144 | 145 | public async deleteOne(filters: IBaseFilters): Promise { 146 | try { 147 | const internalFilters = mapInternalFilters(filters); 148 | 149 | return this.repository.deleteOne(internalFilters); 150 | } catch (error) { 151 | throw new Error(error); 152 | } 153 | } 154 | 155 | public async deleteOneById(uuid: string): Promise { 156 | try { 157 | const internalFilters = mapInternalFilters({ uuid }); 158 | 159 | return this.repository.deleteOne(internalFilters); 160 | } catch (error) { 161 | throw new Error(error); 162 | } 163 | } 164 | } 165 | 166 | export default BaseService; 167 | -------------------------------------------------------------------------------- /src/database/service/index.ts: -------------------------------------------------------------------------------- 1 | import BaseService from "./base"; 2 | import PackageService from "./package"; 3 | import ManifestService from "./manifest"; 4 | import StatsService from "./stats"; 5 | 6 | export { 7 | BaseService, 8 | PackageService, 9 | ManifestService, 10 | StatsService, 11 | }; 12 | -------------------------------------------------------------------------------- /src/database/service/manifest.ts: -------------------------------------------------------------------------------- 1 | import { getMongoRepository } from "typeorm"; 2 | 3 | import BaseService from "./base"; 4 | import { ManifestModel } from "../model"; 5 | import { 6 | IManifest, 7 | IBaseInsert, 8 | IBaseUpdate, 9 | IBaseFilters, 10 | SortOrder, 11 | } from "../types"; 12 | import { mapInternalFilters, escapeRegex } from "../helpers"; 13 | 14 | // yes ik, dependency injection, i swear i will later 15 | class ManifestService extends BaseService { 16 | protected repository = getMongoRepository(ManifestModel); 17 | 18 | // index stuff 19 | public async setupIndices(): Promise { 20 | await this.repository.createCollectionIndex({ 21 | Id: 1, 22 | Version: 1, 23 | }, { 24 | unique: true, 25 | }); 26 | } 27 | 28 | // checks if there is a manifest with the given id and version (should only be 1) 29 | // returns the manifest (if existent) or null (if non-existent) 30 | public async findManifestVersion(id: string, version: string): Promise | undefined> { 31 | const manifest = await this.findOne({ 32 | filters: { 33 | Id: id, 34 | Version: version, 35 | }, 36 | }); 37 | 38 | delete manifest?._id; 39 | 40 | return manifest; 41 | } 42 | 43 | // inserts a manifest into the database 44 | public async insertManifest(manifest: IBaseInsert): Promise { 45 | await this.insertOne(manifest); 46 | } 47 | 48 | // update the fields on all manifests with the given id 49 | public async updateManifests(id: string, updateFields: IBaseUpdate): Promise { 50 | await this.updateOne({ 51 | filters: { 52 | Id: id, 53 | }, 54 | update: updateFields, 55 | }); 56 | } 57 | 58 | // update the fields on a single manifest with the given version and id 59 | public async updateManifestVersion(id: string, version: string, updateFields: IBaseUpdate): Promise { 60 | await this.updateOne({ 61 | filters: { 62 | Id: id, 63 | Version: version, 64 | }, 65 | update: updateFields, 66 | }); 67 | } 68 | 69 | // create a manifest if it doesnt already exist (matched id and version), otherwise update the existing one 70 | public async upsertManifest(manifest: IBaseInsert): Promise { 71 | const { Id: id, Version: version } = manifest; 72 | 73 | // shitty validation (until i do better validation) for something that can 74 | // potentially seriously fuck shit up 75 | if (id == null || version == null) { 76 | throw new Error("id and/or manifest version not set"); 77 | } 78 | 79 | await this.repository.updateOne( 80 | { 81 | Id: id, 82 | Version: version, 83 | }, 84 | { 85 | $set: manifest, 86 | }, 87 | { 88 | upsert: true, 89 | }, 90 | ); 91 | } 92 | 93 | // remove all manifests with the given id 94 | public async removeManifests(id: string): Promise { 95 | await this.delete({ 96 | Id: id, 97 | }); 98 | } 99 | 100 | // remove a single manifest that matched the given id and version 101 | public async removeManifestVersion(id: string, version: string): Promise { 102 | await this.deleteOne({ 103 | Id: id, 104 | Version: version, 105 | }); 106 | } 107 | 108 | // *** legacy package stuff *** 109 | 110 | // TODO: this doesnt return all fields on PackageModel, reflect that in the typings 111 | // TODO: also especially the stuff with diff versions is totally unlike the model types lmao 112 | // TODO: remove the anys 113 | // TODO: we shouldnt need the Version toString converstion, this is probably because: 114 | // input data is fucked, so fix the input, check why tf our validation didnt work, and remove the conversions 115 | // TODO: sort should not be string, it should be limited to fields on PackageModel 116 | // (give or take a few), also make the default a const and move it or make the opts thing an object <--- THIS 117 | // TODO: allow multiple field sort 118 | // TODO: remove this max len thing 119 | private async findPackages( 120 | filters: IBaseFilters, 121 | take: number, 122 | skip = 0, 123 | sort = "Name", 124 | order = SortOrder.ASCENDING, 125 | ): Promise<[ManifestModel[], number]> { 126 | // 127 | try { 128 | const internalFilters = mapInternalFilters(filters); 129 | 130 | const result = await this.repository.aggregate([ 131 | { 132 | $match: internalFilters, 133 | }, 134 | { 135 | $addFields: { 136 | semver: { 137 | $reduce: { 138 | input: { 139 | $map: { 140 | input: { 141 | $map: { 142 | // if version has less than 4 parts, set the remaining ones to 0 143 | input: { 144 | $zip: { 145 | inputs: [ 146 | { 147 | $split: [ 148 | { 149 | $convert: { 150 | input: "$Version", 151 | to: "string", 152 | }, 153 | }, 154 | ".", 155 | ], 156 | }, 157 | { 158 | $range: [ 159 | 0, 160 | 4, 161 | ], 162 | }, 163 | ], 164 | useLongestLength: true, 165 | }, 166 | }, 167 | as: "temp", 168 | in: { 169 | $ifNull: [ 170 | { 171 | $arrayElemAt: [ 172 | "$$temp", 173 | 0, 174 | ], 175 | }, 176 | "0", 177 | ], 178 | }, 179 | }, 180 | }, 181 | as: "ver", 182 | in: { 183 | $concat: [ 184 | { 185 | // get pad string, then pad each ver 186 | $reduce: { 187 | input: { 188 | $range: [ 189 | 0, 190 | { 191 | $subtract: [ 192 | // pad to the longest possible len 193 | // should be 5 chars per section (https://github.com/microsoft/winget-cli/blob/master/doc/ManifestSpecv0.1.md) 194 | // but apparently some peeps think that standards dont apply to them... 195 | // (https://github.com/microsoft/winget-pkgs/blob/master/manifests/Microsoft/dotnet/5.0.100-preview.4.yaml) 196 | // what a fucking cunt (also the utf-16 encoding wtf) 197 | 5, 198 | { 199 | $strLenCP: { 200 | $convert: { 201 | input: "$$ver", 202 | to: "string", 203 | }, 204 | }, 205 | }, 206 | ], 207 | }, 208 | ], 209 | }, 210 | initialValue: "", 211 | in: { 212 | $concat: [ 213 | "$$value", 214 | "0", 215 | ], 216 | }, 217 | }, 218 | }, 219 | "$$ver", 220 | ], 221 | }, 222 | }, 223 | }, 224 | initialValue: "", 225 | // will leave a . at the end but thats fine for our purposes (sorting) 226 | in: { 227 | $concat: [ 228 | "$$value", 229 | "$$this", 230 | ".", 231 | ], 232 | }, 233 | }, 234 | // gets the length of the longest part (wont work cos wed need the global longest el len) 235 | // not worth the effort and imma assume the shit will follow standards 236 | // $max: { 237 | // $map: { 238 | // input: { 239 | // $split: [ 240 | // { 241 | // $convert: { 242 | // input: "$Version", 243 | // to: "string", 244 | // }, 245 | // }, 246 | // ".", 247 | // ], 248 | // }, 249 | // in: { 250 | // $strLenCP: "$$this", 251 | // }, 252 | // }, 253 | // }, 254 | }, 255 | }, 256 | }, 257 | { 258 | $sort: { 259 | semver: -1, 260 | }, 261 | }, 262 | { 263 | $group: { 264 | _id: "$Id", 265 | versions: { 266 | $push: "$Version", 267 | }, 268 | latest: { 269 | $first: "$$ROOT", 270 | }, 271 | }, 272 | }, 273 | { 274 | $project: { 275 | _id: 0, 276 | Id: "$_id", 277 | versions: "$versions", 278 | latest: "$latest", 279 | }, 280 | }, 281 | // get total count of search results 282 | { 283 | $facet: { 284 | total: [ 285 | { 286 | $count: "value", 287 | }, 288 | ], 289 | packages: [ 290 | { 291 | $sort: { 292 | [`latest.${sort}`]: order, 293 | Id: -1, 294 | }, 295 | }, 296 | { 297 | $skip: skip * take, 298 | }, 299 | { 300 | $limit: take, 301 | }, 302 | ], 303 | }, 304 | }, 305 | // clean up latest 306 | // TODO: these dont actually get deleted 307 | { 308 | $unset: [ 309 | "latest._id", 310 | "latest.Id", 311 | 312 | // temp value used for sorting by version 313 | "latest.semver", 314 | ], 315 | }, 316 | // clean up total 317 | { 318 | $unwind: "$total", 319 | }, 320 | { 321 | $project: { 322 | total: "$total.value", 323 | packages: "$packages", 324 | }, 325 | }, 326 | ]).toArray(); 327 | 328 | return [result[0]?.packages ?? [], result[0]?.total ?? 0]; 329 | } catch (error) { 330 | throw new Error(error); 331 | } 332 | } 333 | 334 | // TODO: add an option to pass in the fields that you want on the package part of the response (and get rid of those dorty maps) 335 | public async findAutocomplete(query: string, take: number): Promise { 336 | try { 337 | // TODO: remove the any (part of todos from above) 338 | const results = Promise.all( 339 | [ 340 | this.findPackages({ Name: new RegExp(`.*${escapeRegex(query)}.*`, "i") }, take), 341 | this.findPackages({ Publisher: new RegExp(`.*${escapeRegex(query)}.*`, "i") }, take), 342 | this.findPackages({ Description: new RegExp(`.*${escapeRegex(query)}.*`, "i") }, take), 343 | ], 344 | ).then(e => e 345 | .flatMap(f => f[0]) 346 | .slice(0, take) 347 | .filter((f, i, a) => a.findIndex(g => g.Id === f.Id) === i) 348 | .map((f: any) => ({ 349 | ...f, 350 | latest: { 351 | Version: f.latest.Version, 352 | Name: f.latest.Name, 353 | Publisher: f.latest.Publisher, 354 | Description: f.latest.Description, 355 | Homepage: f.latest.Homepage, 356 | // IconUrl: f.latest.IconUrl, 357 | }, 358 | }))); 359 | 360 | return results; 361 | } catch (error) { 362 | throw new Error(error); 363 | } 364 | } 365 | 366 | // TODO: sort should not be string, it should be limited to fields on PackageModel (give or take a few) 367 | public async findByName(name: string, take: number, skip: number, sort: string, order: number): Promise<[ManifestModel[], number]> { 368 | const [packages, total] = await this.findPackages({ Name: new RegExp(`.*${escapeRegex(name)}.*`, "i") }, take, skip, sort, order); 369 | 370 | const packageBasicInfo = packages.map((f: any) => ({ 371 | ...f, 372 | latest: { 373 | Version: f.latest.Version, 374 | Name: f.latest.Name, 375 | Publisher: f.latest.Publisher, 376 | Description: f.latest.Description, 377 | Homepage: f.latest.Homepage, 378 | // IconUrl: f.latest.IconUrl, 379 | }, 380 | })); 381 | 382 | return [packageBasicInfo, total]; 383 | } 384 | 385 | public async findByOrg(org: string, take: number, skip: number): Promise<[ManifestModel[], number]> { 386 | const [packages, total] = await this.findPackages({ Id: new RegExp(`^${escapeRegex(org)}\\..*`, "i") }, take, skip); 387 | 388 | const packageBasicInfo = packages.map((f: any) => ({ 389 | ...f, 390 | latest: { 391 | Version: f.latest.Version, 392 | Name: f.latest.Name, 393 | Publisher: f.latest.Publisher, 394 | Description: f.latest.Description, 395 | Homepage: f.latest.Homepage, 396 | // IconUrl: f.latest.IconUrl, 397 | }, 398 | })); 399 | 400 | return [packageBasicInfo, total]; 401 | } 402 | 403 | public async findByPackage(org: string, pkg: string): Promise { 404 | const [packages] = await this.findPackages({ Id: new RegExp(`^${escapeRegex(org)}\\.${escapeRegex(pkg)}$`, "i") }, 1); 405 | 406 | return packages[0]; 407 | } 408 | } 409 | 410 | export default ManifestService; 411 | -------------------------------------------------------------------------------- /src/database/service/package.ts: -------------------------------------------------------------------------------- 1 | import { getMongoRepository } from "typeorm"; 2 | 3 | import BaseService from "./base"; 4 | import PackageModel from "../model/package"; 5 | import { 6 | generateMetaphones, 7 | generateNGrams, 8 | dedupe, 9 | escapeRegex, 10 | } from "../helpers"; 11 | import { 12 | IPackage, 13 | IBaseInsert, 14 | IPackageQueryOptions, 15 | SortOrder, 16 | PackageSortFields, 17 | IPackageSearchOptions, 18 | } from "../types"; 19 | 20 | const { 21 | NODE_ENV, 22 | } = process.env; 23 | 24 | class PackageService extends BaseService { 25 | repository = getMongoRepository(PackageModel); 26 | 27 | // index stuff 28 | public async setupIndices(): Promise { 29 | await this.repository.createCollectionIndexes([ 30 | { 31 | key: { 32 | Id: 1, 33 | }, 34 | unique: true, 35 | }, 36 | { 37 | key: { 38 | "Search.Name": "text", 39 | "Search.Publisher": "text", 40 | "Search.Tags": "text", 41 | "Search.Description": "text", 42 | }, 43 | // will probably always match the name first 44 | weights: { 45 | "Search.Name": 20, 46 | "Search.Publisher": 12, 47 | "Search.Tags": 10, 48 | "Search.Description": 4, 49 | }, 50 | }, 51 | ]); 52 | } 53 | 54 | // create a package if it doesnt already exist (matched id), otherwise update the existing one 55 | public async upsertPackage(pkg: IBaseInsert): Promise { 56 | const { Id: id } = pkg; 57 | 58 | // shitty validation (until i do better validation) for something that can 59 | // potentially seriously fuck shit up 60 | if (id == null) { 61 | throw new Error("id not set"); 62 | } 63 | 64 | await this.repository.updateOne( 65 | { 66 | Id: id, 67 | }, 68 | { 69 | $set: pkg, 70 | }, 71 | { 72 | upsert: true, 73 | }, 74 | ); 75 | } 76 | 77 | public async searchPackages( 78 | queryOptions: IPackageQueryOptions, 79 | take: number, 80 | page: number, 81 | sort: PackageSortFields | "SearchScore", 82 | order: SortOrder, 83 | searchOptions: IPackageSearchOptions, 84 | ): Promise<[Omit[], number]> { 85 | // 86 | const optionFields = Object.values(queryOptions).filter(e => e != null); 87 | 88 | // error if query AND another field is set (query only requirement) 89 | if (queryOptions.query != null && optionFields.length > 1) { 90 | throw new Error("no other queryOptions should be set when 'query' is non-null"); 91 | } 92 | 93 | // error if non-query fields are set and ensureContains is false, in which case 94 | // all non-query fields behave like a qquery and may be misleading 95 | if (queryOptions.query == null && optionFields.length >= 1 && searchOptions.ensureContains === false) { 96 | throw new Error("non-query search parameters are redundant if ensureContains is false"); 97 | } 98 | 99 | // dont run the complicated shit if theres no need to 100 | if (optionFields.length === 0) { 101 | const allPkgs = await this.repository.findAndCount({ 102 | take, 103 | skip: page * take, 104 | // SearchScore doesnt apply here (were not doing any searching) 105 | ...(sort === "SearchScore" ? {} : { 106 | order: { 107 | [sort]: order, 108 | }, 109 | }), 110 | }); 111 | 112 | // NOTE: i want to have a search score set to 0 rather than it possibly being 113 | // undefined, its just easier to work with for anyone consuming the api 114 | return [ 115 | allPkgs[0].map(e => ({ 116 | ...e, 117 | // TODO: this is broke, fix when fixing the uuid issue 118 | uuid: undefined as unknown as string, 119 | 120 | _id: undefined, 121 | SearchScore: 0, 122 | })), 123 | allPkgs[1], 124 | ]; 125 | } 126 | 127 | // NOTE: unlike generateNGrams, i dont want to edit the generateMetaphones fn itself as other ones 128 | // call it so something is likely to break (and i dont have unit tests for it so...) 129 | // in the future, it would be wise to have general useful fns and ones which transform the results 130 | // of those to a less universally usable format (like im doing here with the '_') 131 | const processQueryInput = searchOptions.partialMatch === true 132 | ? (word: string): string[] => generateNGrams(word, 2) 133 | : (word: string): string[] => generateMetaphones(word).map(e => e.padEnd(3, "_")); 134 | 135 | const query = dedupe(optionFields.flat().map((e: string) => { 136 | if (searchOptions.splitQuery === true) { 137 | return e.split(" ").map(f => processQueryInput(f)).flat(); 138 | } 139 | return processQueryInput(e); 140 | }).flat()).join(" "); 141 | 142 | // these vars can probs be set in a much nicer way (refactoring) 143 | const nameQuery = queryOptions.query ?? queryOptions.name ?? ""; 144 | const publisherQuery = queryOptions.query ?? queryOptions.publisher ?? ""; 145 | const descriptionQuery = queryOptions.query ?? queryOptions.description ?? ""; 146 | 147 | const nameRegex = new RegExp(`.*(${ 148 | searchOptions.splitQuery === false ? escapeRegex(nameQuery) : nameQuery.split(" ").map(e => escapeRegex(e)).join("|") 149 | }).*`, "i"); 150 | const publisherRegex = new RegExp(`.*(${ 151 | searchOptions.splitQuery === false ? escapeRegex(publisherQuery) : publisherQuery.split(" ").map(e => escapeRegex(e)).join("|") 152 | }).*`, "i"); 153 | const descriptionRegex = new RegExp(`.*(${ 154 | searchOptions.splitQuery === false ? escapeRegex(descriptionQuery) : descriptionQuery.split(" ").map(e => escapeRegex(e)).join("|") 155 | }).*`, "i"); 156 | 157 | const pkgs = await this.repository.aggregate([ 158 | { 159 | $match: { 160 | $and: [ 161 | { 162 | $text: { 163 | $search: query, 164 | }, 165 | }, 166 | ...(searchOptions.ensureContains === false ? [] : [ 167 | { 168 | // NOTE: tags will be used to bump up the weight of searches but wont be 169 | // checked against a regex, would have to exact match to make sense and thats 170 | // just not worth it (perf vs how much it would improve results) 171 | $or: [ 172 | // at least 1 should be set unless someone fucked up 173 | // NOTE: the value passed into escapeRegex should never be null without the '?? ""' but ts complained 174 | ...((queryOptions.query ?? queryOptions.name) == null ? [] : [ 175 | { 176 | "Latest.Name": { 177 | $regex: nameRegex, 178 | }, 179 | }, 180 | ]), 181 | ...((queryOptions.query ?? queryOptions.publisher) == null ? [] : [ 182 | { 183 | "Latest.Publisher": { 184 | $regex: publisherRegex, 185 | }, 186 | }, 187 | ]), 188 | // exact match for tags 189 | // NOTE: im doing lowerCase stuff all over the place currently, in the future it would be better 190 | // to do it in the api routes themselves and leave the db logic universal, so one api ver can be 191 | // case sensitive/insensitive if something like that is required for example 192 | ...((queryOptions.query ?? queryOptions.tags) == null ? [] : [ 193 | { 194 | "Latest.Tags": { 195 | $in: queryOptions.query != null ? [queryOptions.query.toLowerCase()] : queryOptions.tags?.map(e => e.toLowerCase()), 196 | }, 197 | }, 198 | ]), 199 | ...((queryOptions.query ?? queryOptions.description) == null ? [] : [ 200 | { 201 | "Latest.Description": { 202 | $regex: descriptionRegex, 203 | }, 204 | }, 205 | ]), 206 | ], 207 | }, 208 | ]), 209 | ], 210 | }, 211 | }, 212 | { 213 | $facet: { 214 | total: [ 215 | { 216 | $count: "count", 217 | }, 218 | ], 219 | results: [ 220 | { 221 | $sort: { 222 | ...(sort === "SearchScore" ? { 223 | score: { 224 | $meta: "textScore", 225 | }, 226 | } : { 227 | [sort]: order, 228 | }), 229 | // in case there are multiple docs with the same date 230 | Id: -1, 231 | }, 232 | }, 233 | { 234 | $skip: page * take, 235 | }, 236 | { 237 | $limit: take, 238 | }, 239 | { 240 | $unset: [ 241 | "_id", 242 | ...(NODE_ENV === "dev" ? [] : [ 243 | "Search", 244 | ]), 245 | ], 246 | }, 247 | { 248 | $addFields: { 249 | SearchScore: { 250 | $meta: "textScore", 251 | }, 252 | }, 253 | }, 254 | ], 255 | }, 256 | }, 257 | { 258 | $unwind: "$total", 259 | }, 260 | { 261 | $replaceRoot: { 262 | newRoot: { 263 | total: "$total.count", 264 | results: "$results", 265 | }, 266 | }, 267 | }, 268 | ]).next(); 269 | 270 | return [pkgs?.results ?? [], pkgs?.total ?? 0]; 271 | } 272 | 273 | // NOTE: shitty typeorm types 274 | // yes i have to use the aggregation pipeline unfortunately, cunty typeorm returns documents 275 | // even if they dont match if i specify any of take, skip, order, etc. actually fucking RETARDED 276 | public async findByPublisher( 277 | publisher: string, 278 | take: number, 279 | page: number, 280 | sort: PackageSortFields, 281 | order: SortOrder, 282 | ): Promise<[Omit[], number]> { 283 | // 284 | const pkgs = await this.repository.aggregate([ 285 | { 286 | $match: { 287 | Id: { 288 | $regex: new RegExp(`^${publisher}\\.`), 289 | }, 290 | }, 291 | }, 292 | { 293 | $facet: { 294 | total: [ 295 | { 296 | $count: "count", 297 | }, 298 | ], 299 | results: [ 300 | { 301 | $sort: { 302 | [sort]: order, 303 | }, 304 | }, 305 | { 306 | $skip: page * take, 307 | }, 308 | { 309 | $limit: take, 310 | }, 311 | { 312 | $unset: "_id", 313 | }, 314 | ], 315 | }, 316 | }, 317 | { 318 | $unwind: "$total", 319 | }, 320 | { 321 | $replaceRoot: { 322 | newRoot: { 323 | total: "$total.count", 324 | results: "$results", 325 | }, 326 | }, 327 | }, 328 | ]).next(); 329 | 330 | return [pkgs?.results ?? [], pkgs?.total ?? 0]; 331 | } 332 | 333 | public async findSinglePackage(publisher: string, packageName: string): Promise | undefined> { 334 | const pkg = await this.repository.findOne({ 335 | Id: `${publisher}.${packageName}`, 336 | }); 337 | 338 | delete pkg?._id; 339 | 340 | return pkg; 341 | } 342 | } 343 | 344 | export default PackageService; 345 | -------------------------------------------------------------------------------- /src/database/service/stats.ts: -------------------------------------------------------------------------------- 1 | import { getMongoRepository } from "typeorm"; 2 | import moment from "moment"; 3 | 4 | import BaseService from "./base"; 5 | import { StatsModel } from "../model"; 6 | import { StatsResolution, IStatsSeries, IStats } from "../types"; 7 | 8 | // 1 hour 9 | // DO NOT CHANGE THIS!!! (unless you know what youre doing lol) 10 | const SAMPLING_PERIOD = 1000 * 60 * 60 * 24; 11 | 12 | class StatsService extends BaseService { 13 | repository = getMongoRepository(StatsModel); 14 | 15 | public async incrementAccessCount(packageId: string): Promise { 16 | const currentPeriod = Math.floor(Date.now() / SAMPLING_PERIOD); 17 | 18 | await this.repository.updateOne( 19 | { 20 | PackageId: packageId, 21 | Period: currentPeriod, 22 | }, 23 | { 24 | $set: { 25 | PackageId: packageId, 26 | Period: currentPeriod, 27 | }, 28 | $inc: { 29 | Value: 1, 30 | }, 31 | }, 32 | { 33 | upsert: true, 34 | }, 35 | ); 36 | } 37 | 38 | // NOTE: before and after are inclusive of the supplied dates 39 | public async getPackageStats(packageId: string, resolution: StatsResolution, after: Date, before: Date): Promise { 40 | const startPeriod = Math.floor(after.getTime() / SAMPLING_PERIOD); 41 | const endPeriod = Math.floor(before.getTime() / SAMPLING_PERIOD); 42 | 43 | // need to do this due to bad typings in typeorm 44 | // TODO: also yes ik mongo can sort shit, typeorm has no docs for this and i cant be fucked rn 45 | const stats = await this.repository.find({ 46 | PackageId: packageId, 47 | Period: { 48 | $gte: startPeriod, 49 | $lte: endPeriod, 50 | } as unknown as number, 51 | }); 52 | 53 | stats.sort((a, b) => a.Period - b.Period); 54 | 55 | const grouped: { [key: string]: IStats[] } = {}; 56 | stats.forEach(stat => { 57 | const time = moment(stat.Period * SAMPLING_PERIOD).utc().startOf(resolution).toISOString(); 58 | 59 | grouped[time] = [...grouped[time] ?? [], stat]; 60 | }); 61 | 62 | // [time, stat[]] 63 | const series = Object.entries(grouped).map(([k, v]) => ({ 64 | Period: new Date(k), 65 | Value: v.reduce((a, c) => a + c.Value, 0), 66 | })); 67 | 68 | return series; 69 | } 70 | } 71 | 72 | export default StatsService; 73 | -------------------------------------------------------------------------------- /src/database/types/base.ts: -------------------------------------------------------------------------------- 1 | // TODO: readonly shite here and in other interfaces (where applicable) 2 | // TODO: remove select for shit like _id and __v, etc 3 | interface IBase { 4 | // TODO: make the _id type more specific 5 | _id: object; 6 | uuid: string; 7 | __v: number; 8 | 9 | createdAt: Date; 10 | updatedAt: Date; 11 | } 12 | 13 | export { 14 | // eslint-disable-next-line import/prefer-default-export 15 | IBase, 16 | }; 17 | -------------------------------------------------------------------------------- /src/database/types/error.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/winget-run/api/a99b971caf0d12a34d1bfd69e711900fba8169a9/src/database/types/error.ts -------------------------------------------------------------------------------- /src/database/types/index.ts: -------------------------------------------------------------------------------- 1 | import { IBase } from "./base"; 2 | // import {} from "./error"; 3 | import { IManifest } from "./manifest"; 4 | import { StatsResolution, IStats, IStatsSeries } from "./stats"; 5 | import { 6 | IPackage, 7 | IPackageQueryOptions, 8 | IPackageSearchOptions, 9 | PackageSortFields, 10 | } from "./package"; 11 | import { 12 | SortOrder, 13 | IBaseFilters, 14 | IBaseInternalFilters, 15 | IBaseSelect, 16 | IBaseOrder, 17 | IBaseFindOneOptions, 18 | IBaseFindManyOptions, 19 | IBaseInternalFindOneOptions, 20 | IBaseInternalFindManyOptions, 21 | IBaseInsert, 22 | IInsertResult, 23 | IBaseUpdate, 24 | IBaseUpdateOptions, 25 | IUpdateResult, 26 | IDeleteResult, 27 | } from "./service"; 28 | 29 | export { 30 | IBase, 31 | IPackage, 32 | IPackageQueryOptions, 33 | IPackageSearchOptions, 34 | PackageSortFields, 35 | IManifest, 36 | StatsResolution, 37 | IStats, 38 | IStatsSeries, 39 | 40 | SortOrder, 41 | IBaseFilters, 42 | IBaseInternalFilters, 43 | IBaseSelect, 44 | IBaseOrder, 45 | IBaseFindOneOptions, 46 | IBaseFindManyOptions, 47 | IBaseInternalFindOneOptions, 48 | IBaseInternalFindManyOptions, 49 | IBaseInsert, 50 | IInsertResult, 51 | IBaseUpdate, 52 | IBaseUpdateOptions, 53 | IUpdateResult, 54 | IDeleteResult, 55 | }; 56 | -------------------------------------------------------------------------------- /src/database/types/manifest.ts: -------------------------------------------------------------------------------- 1 | import { IBase } from "./base"; 2 | 3 | interface IManifest extends IBase { 4 | Id: string; 5 | Name: string; 6 | AppMoniker?: string; 7 | Version: string; 8 | Publisher: string; 9 | Channel?: string; 10 | Author?: string; 11 | License?: string; 12 | LicenseUrl?: string; 13 | MinOSVersion?: string; 14 | Description?: string; 15 | Homepage?: string; 16 | Tags?: string; 17 | FileExtensions?: string; 18 | Protocols?: string; 19 | Commands?: string; 20 | InstallerType?: string; 21 | Switches?: { 22 | Custom?: string; 23 | Silent?: string; 24 | SilentWithProgress?: string; 25 | Interactive?: string; 26 | Language?: string; 27 | }; 28 | Log?: string; 29 | InstallLocation?: string; 30 | Installers: [ 31 | { 32 | Arch: string; 33 | Url: string; 34 | Sha256: string; 35 | SignatureSha256?: string; 36 | Language?: string; 37 | InstallerType?: string; 38 | Scope?: string; 39 | SystemAppId?: string; 40 | Switches?: { 41 | Language?: string; 42 | Custom?: string; 43 | }; 44 | } 45 | ]; 46 | Localization?: [ 47 | { 48 | Language: string; 49 | Description?: string; 50 | Homepage?: string; 51 | LicenseUrl?: string; 52 | } 53 | ]; 54 | } 55 | 56 | export { 57 | // eslint-disable-next-line import/prefer-default-export 58 | IManifest, 59 | }; 60 | -------------------------------------------------------------------------------- /src/database/types/package.ts: -------------------------------------------------------------------------------- 1 | import { IBase } from "./base"; 2 | 3 | // TODO validation and enums 4 | interface IPackage extends IBase { 5 | Id: string; 6 | 7 | // version stuff 8 | Versions: string[]; 9 | Latest: { 10 | Name: string; 11 | Publisher: string; 12 | Tags: string[]; 13 | Description?: string; 14 | Homepage?: string; 15 | License?: string; 16 | LicenseUrl?: string; 17 | }; 18 | 19 | // extra 20 | Featured: boolean; 21 | IconUrl?: string; 22 | Banner?: string; 23 | Logo?: string; 24 | 25 | // search 26 | Search: { 27 | Name: string; 28 | Publisher: string; 29 | Tags?: string; 30 | Description?: string; 31 | }; 32 | 33 | // stats 34 | UpdatedAt: Date; 35 | CreatedAt: Date; 36 | } 37 | 38 | interface IPackageQueryOptions { 39 | query?: string; 40 | name?: string; 41 | publisher?: string; 42 | description?: string; 43 | tags?: string[]; 44 | } 45 | 46 | interface IPackageSearchOptions { 47 | splitQuery?: boolean; 48 | partialMatch?: boolean; 49 | ensureContains?: boolean; 50 | } 51 | 52 | enum PackageSortFields { 53 | LatestName = "Latest.Name", 54 | LatestPublisher = "Latest.Publisher", 55 | UpdatedAt = "UpdatedAt", 56 | } 57 | 58 | export { 59 | IPackage, 60 | IPackageQueryOptions, 61 | IPackageSearchOptions, 62 | PackageSortFields, 63 | }; 64 | -------------------------------------------------------------------------------- /src/database/types/service.ts: -------------------------------------------------------------------------------- 1 | import { FindManyOptions, FindOneOptions } from "typeorm"; 2 | 3 | import { IBase } from "./base"; 4 | 5 | enum SortOrder { 6 | ASCENDING = 1, 7 | DESCENDING = -1, 8 | } 9 | 10 | // type IBaseFilters = Partial>; 11 | // type IBaseInternalFilters = Partial>; 12 | 13 | // string | anythingElse 14 | // string | regex | anythingElse 15 | 16 | // NOTE: this is as close to sane as i could get it 17 | type IBaseFilters = { 18 | [P in keyof Omit]?: NonNullable extends string ? T[P] | RegExp : T[P]; 19 | } & { uuid?: string }; 20 | 21 | type IBaseInternalFilters = { 22 | [P in keyof Omit]?: NonNullable extends string ? T[P] | RegExp : T[P]; 23 | }; 24 | 25 | type IBaseSelect = (keyof T)[]; 26 | type IBaseOrder> = { 27 | [P in keyof T]?: SortOrder; 28 | }; 29 | 30 | interface IBaseFindOneOptions { 31 | filters: IBaseFilters; 32 | select?: IBaseSelect; 33 | order?: IBaseOrder; 34 | } 35 | 36 | type IBaseFindManyOptions = IBaseFindOneOptions & { 37 | take?: number; 38 | skip?: number; 39 | } 40 | 41 | type IBaseInternalFindOneOptions = FindOneOptions; 42 | type IBaseInternalFindManyOptions = FindManyOptions; 43 | 44 | type IBaseInsert = Omit; 45 | 46 | interface IInsertResult { 47 | insertedCount: number; 48 | } 49 | 50 | type IBaseUpdate = Partial>; 51 | 52 | interface IBaseUpdateOptions { 53 | filters: IBaseFilters; 54 | update: IBaseUpdate; 55 | } 56 | 57 | interface IUpdateResult { 58 | matchedCount: number; 59 | modifiedCount: number; 60 | } 61 | 62 | interface IDeleteResult { 63 | deletedCount?: number; 64 | } 65 | 66 | export { 67 | SortOrder, 68 | IBaseFilters, 69 | IBaseInternalFilters, 70 | IBaseSelect, 71 | IBaseOrder, 72 | IBaseFindOneOptions, 73 | IBaseFindManyOptions, 74 | IBaseInternalFindOneOptions, 75 | IBaseInternalFindManyOptions, 76 | IBaseInsert, 77 | IInsertResult, 78 | IBaseUpdate, 79 | IBaseUpdateOptions, 80 | IUpdateResult, 81 | IDeleteResult, 82 | }; 83 | -------------------------------------------------------------------------------- /src/database/types/stats.ts: -------------------------------------------------------------------------------- 1 | import { IBase } from "."; 2 | 3 | enum StatsResolution { 4 | // Hour = "hour", 5 | Day = "day", 6 | Week = "isoWeek", 7 | Month = "month", 8 | } 9 | 10 | interface IStats extends IBase { 11 | PackageId: string; 12 | Period: number; 13 | Value: number; 14 | } 15 | 16 | interface IStatsSeries { 17 | Period: Date; 18 | Value: number; 19 | } 20 | 21 | export { 22 | StatsResolution, 23 | IStats, 24 | IStatsSeries, 25 | }; 26 | -------------------------------------------------------------------------------- /src/env.d.ts: -------------------------------------------------------------------------------- 1 | declare namespace NodeJS { 2 | export interface ProcessEnv { 3 | NODE_ENV: "prod" | "dev" | "test"; 4 | NODE_PATH: string; 5 | 6 | MONGO_HOST: string; 7 | MONGO_DB: string; 8 | MONGO_CERT: string; 9 | MONGO_CA: string; 10 | MONGO_CA_PATH: string; 11 | 12 | WEB_ADDRESS: string; 13 | WEBSERVER_LOGGER: string; 14 | WEBSERVER_PORT: string; 15 | WEBSERVER_ADDRESS: string; 16 | 17 | GITHUB_TOKEN: string; 18 | CRON_FREQUENCY: string; 19 | UPDATE_FREQUENCY_MINUTES: string; 20 | 21 | API_ACCESS_TOKEN: string; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import "reflect-metadata"; 2 | 3 | import { connect } from "./database"; 4 | import { startServer } from "./server"; 5 | 6 | // TODO: add a way to quickly deploy changes if only the /chart folder changes 7 | // TODO: go through all db stuff and make sure it works and returns the correct shit 8 | // TODO: process.env type definitions + runtime type checking 9 | // TODO: all the testing 10 | // TODO: migrations? 11 | // TODO: importance of resilience and proper error handling (were dealing with shitty user uploaded data), 12 | // nothing should go down if half the shit catches fire 13 | // TODO: yarn plug n play 14 | // TODO: restrict access to dev-*.winget.run !important; 15 | // TODO: a script/cron job? which mirrors prod -> dev db 16 | // TODO: make it so more than 1 person can use telepresence at once 17 | // TODO: when we restrict dev, make it always run on NODE_ENV=dev 18 | // TODO: make a public devops repo (minus secrets), and a separate secrets repo, also clean up the devops shite! 19 | (async (): Promise => { 20 | try { 21 | await connect(); 22 | await startServer(); 23 | } catch (error) { 24 | console.error(`startup error: ${error}`); 25 | process.exit(-1); 26 | } 27 | })(); 28 | -------------------------------------------------------------------------------- /src/server/buildFastify.ts: -------------------------------------------------------------------------------- 1 | import Fastify, { FastifyInstance } from "fastify"; 2 | import fastifyCors from "fastify-cors"; 3 | 4 | import routes from "./routes"; 5 | 6 | const { 7 | WEB_ADDRESS, 8 | } = process.env; 9 | 10 | const buildFastify = (settings = {}): FastifyInstance => { 11 | const fastify = Fastify(settings); 12 | 13 | // TODO: this doesnt work for matt, big thonk 14 | fastify.register(fastifyCors, { 15 | origin: WEB_ADDRESS, 16 | }); 17 | 18 | fastify.register(routes); 19 | 20 | return fastify; 21 | }; 22 | 23 | export default buildFastify; 24 | -------------------------------------------------------------------------------- /src/server/ghService/helpers/decodingHelper.ts: -------------------------------------------------------------------------------- 1 | import { TextDecoder } from "util"; 2 | import encoding from "encoding-japanese"; 3 | 4 | import * as jsYaml from "js-yaml"; 5 | 6 | // converts encoding.js returned enocding to one understood by node 7 | // need to handle utf-32, binary, unicode, and auto specially 8 | const encodingTable = { 9 | UTF32: "utf-32", 10 | UTF16: "utf-16", 11 | UTF16BE: "utf-16be", 12 | UTF16LE: "utf-16le", 13 | BINARY: "binary", 14 | ASCII: "ascii", 15 | JIS: "iso-2022-jp", 16 | UTF8: "utf-8", 17 | EUCJP: "euc-jp", 18 | SJIS: "sjis", 19 | UNICODE: "unicode", 20 | // extra encoding.js type 21 | AUTO: "auto", 22 | }; 23 | 24 | const parsePackageYaml = async (buf: Buffer): Promise => { 25 | let parsedYaml; 26 | 27 | try { 28 | const detected = encodingTable[encoding.detect(buf)]; 29 | if (["utf-32", "binary", "unicode", "auto"].includes(detected)) { 30 | console.log(`unsupported encoding: ${detected}`); 31 | } 32 | const decoder = new TextDecoder(detected); 33 | 34 | const text = decoder.decode(buf); 35 | 36 | parsedYaml = jsYaml.safeLoad(text); 37 | parsedYaml.Version = String(parsedYaml.Version); 38 | } catch (error) { 39 | console.log(error); 40 | } 41 | 42 | return parsedYaml; 43 | }; 44 | 45 | export { 46 | // eslint-disable-next-line import/prefer-default-export 47 | parsePackageYaml, 48 | }; 49 | -------------------------------------------------------------------------------- /src/server/ghService/import/importPackageUtil.ts: -------------------------------------------------------------------------------- 1 | import { TextDecoder } from "util"; 2 | 3 | import fetch from "node-fetch"; 4 | 5 | import * as jsYaml from "js-yaml"; 6 | import { ManifestFolderList } from "../types/import/manifestFolderListModel"; 7 | 8 | 9 | const { 10 | GITHUB_TOKEN, 11 | } = process.env; 12 | 13 | const CONTENTS_BASE_URL = "https://api.github.com/repos/microsoft/winget-pkgs/contents"; 14 | 15 | //! only call for initial import 16 | const getManifestFolderPaths = async (): Promise => { 17 | const manifestFolderList: Promise = await fetch( 18 | `${CONTENTS_BASE_URL}/manifests`, { 19 | headers: { 20 | Authorization: `token ${GITHUB_TOKEN}`, 21 | }, 22 | }, 23 | ).then((res) => res.json()); 24 | 25 | const manifestFolderPaths = (await manifestFolderList).map((x) => x.path); 26 | 27 | // TODO remove 28 | return manifestFolderPaths; 29 | }; 30 | 31 | const getPackageFolderPaths = async (): Promise => { 32 | //! only use for inital bulk import 33 | const manifestFolderPaths = await getManifestFolderPaths(); 34 | 35 | const packageFolders: ManifestFolderList[] = []; 36 | for (let i = 0; i < manifestFolderPaths.length; i += 1) { 37 | // eslint-disable-next-line no-await-in-loop 38 | const packageFolder = await fetch(`${CONTENTS_BASE_URL}/${manifestFolderPaths[i]}`, { 39 | headers: { 40 | Authorization: `token ${GITHUB_TOKEN}`, 41 | }, 42 | }).then(res => res.json()); 43 | 44 | console.log(packageFolder); 45 | packageFolders.push(packageFolder); 46 | } 47 | 48 | const flatPackageFolderPaths: ManifestFolderList[] = packageFolders.flat( 49 | packageFolders.length, 50 | ); 51 | const packageFolderPaths = flatPackageFolderPaths.map((x) => x.path); 52 | 53 | return packageFolderPaths; 54 | }; 55 | 56 | // for people like mongo who cant read the docs 57 | const handleThreeLevelDeep = async (path: string): Promise => { 58 | const downloadUrlPath: ManifestFolderList[] = await fetch(`${CONTENTS_BASE_URL}/${path}`, { 59 | headers: { 60 | Authorization: `token ${GITHUB_TOKEN}`, 61 | }, 62 | }).then(res => res.json()); 63 | 64 | const downloadUrl = downloadUrlPath[0].download_url; 65 | 66 | return downloadUrl; 67 | }; 68 | 69 | const getPackageDownloadUrls = async (): Promise => { 70 | const packageFolderPaths = await getPackageFolderPaths(); 71 | 72 | const downloadUrlPaths: ManifestFolderList[] = []; 73 | for (let i = 0; i < packageFolderPaths.length; i += 1) { 74 | // eslint-disable-next-line no-await-in-loop 75 | const downloadUrlPath = await fetch(`${CONTENTS_BASE_URL}/${packageFolderPaths[i]}`, { 76 | headers: { 77 | Authorization: `token ${GITHUB_TOKEN}`, 78 | }, 79 | }).then(res => res.json()); 80 | 81 | console.log(downloadUrlPath); 82 | downloadUrlPaths.push(downloadUrlPath); 83 | } 84 | 85 | const flatDownloadUrls: ManifestFolderList[] = downloadUrlPaths.flat( 86 | downloadUrlPaths.length, 87 | ); 88 | 89 | const downloadUrls: string[] = []; 90 | // check if it has three levels 91 | for (let i = 0; i < flatDownloadUrls.length; i += 1) { 92 | if (flatDownloadUrls[i].download_url == null) { 93 | // eslint-disable-next-line no-await-in-loop 94 | const url: string = await handleThreeLevelDeep(flatDownloadUrls[i].path); 95 | downloadUrls.push(url); 96 | } else { 97 | downloadUrls.push(flatDownloadUrls[i].download_url); 98 | } 99 | } 100 | 101 | return downloadUrls; 102 | }; 103 | 104 | const getPackageYamls = async (): Promise => { 105 | const downloadUrls = await (await getPackageDownloadUrls()).filter(x => x != null); 106 | 107 | const packageYamls: string[] = []; 108 | for (let i = 0; i < downloadUrls.length; i += 1) { 109 | // eslint-disable-next-line no-await-in-loop 110 | const packageYaml = await fetch(downloadUrls[i], { 111 | headers: { 112 | Authorization: `token ${GITHUB_TOKEN}`, 113 | }, 114 | }).then(res => res.buffer()) 115 | .then(buffer => { 116 | const utf8decoder = new TextDecoder("utf-8"); 117 | const utf16decoder = new TextDecoder("utf-16"); 118 | 119 | let res; 120 | try { 121 | const text = utf8decoder.decode(buffer); 122 | res = jsYaml.safeLoad(text); 123 | } catch (error) { 124 | const text = utf16decoder.decode(buffer); 125 | res = jsYaml.safeLoad(text); 126 | } 127 | 128 | res.Version = String(res.Version); 129 | return res; 130 | }); 131 | 132 | console.log(packageYaml); 133 | packageYamls.push(packageYaml); 134 | } 135 | 136 | return packageYamls; 137 | }; 138 | 139 | export = { 140 | getPackageYamls, 141 | }; 142 | -------------------------------------------------------------------------------- /src/server/ghService/import/manualImportUtil.ts: -------------------------------------------------------------------------------- 1 | 2 | import fetch from "node-fetch"; 3 | import { TextDecoder } from "util"; 4 | import * as jsYaml from "js-yaml"; 5 | 6 | import { ManifestFolderList } from "../types/import/manifestFolderListModel"; 7 | 8 | const { 9 | GITHUB_TOKEN, 10 | } = process.env; 11 | 12 | const CONTENTS_BASE_URL = "https://api.github.com/repos/microsoft/winget-pkgs/contents"; 13 | 14 | //! only call for initial import 15 | const getManifestFolderPaths = async (manifests: string[]): Promise => { 16 | const manifestFolderPaths = manifests; 17 | 18 | return manifestFolderPaths; 19 | }; 20 | 21 | const getPackageFolderPaths = async (manifests: string[]): Promise => { 22 | const manifestFolderPaths = await getManifestFolderPaths(manifests); 23 | 24 | const packageFolders: ManifestFolderList[] = await Promise.all( 25 | manifestFolderPaths.map((e) => fetch(`${CONTENTS_BASE_URL}/${e}`, { 26 | headers: { 27 | Authorization: `token ${GITHUB_TOKEN}`, 28 | }, 29 | }).then((res) => res.json())), 30 | ); 31 | 32 | const flatPackageFolderPaths: ManifestFolderList[] = packageFolders.flat( 33 | packageFolders.length, 34 | ); 35 | const packageFolderPaths = flatPackageFolderPaths.map((x) => x.path); 36 | 37 | return packageFolderPaths; 38 | }; 39 | 40 | // for people like mongo who cant read the docs 41 | const handleThreeLevelDeep = async (path: string): Promise => { 42 | const downloadUrlPath: ManifestFolderList[] = await fetch(`${CONTENTS_BASE_URL}/${path}`, { 43 | headers: { 44 | Authorization: `token ${GITHUB_TOKEN}`, 45 | }, 46 | }).then(res => res.json()); 47 | 48 | const downloadUrl = downloadUrlPath[0].download_url; 49 | 50 | return downloadUrl; 51 | }; 52 | 53 | 54 | const getPackageDownloadUrls = async (manifests: string[]): Promise => { 55 | const packageFolderPaths = await getPackageFolderPaths(manifests); 56 | 57 | const downloadUrlPaths: ManifestFolderList[] = await Promise.all( 58 | packageFolderPaths.map((path) => fetch(`${CONTENTS_BASE_URL}/${path}`, { 59 | headers: { 60 | Authorization: `token ${GITHUB_TOKEN}`, 61 | }, 62 | }).then((res) => res.json())), 63 | ); 64 | 65 | const flatDownloadUrls: ManifestFolderList[] = downloadUrlPaths.flat( 66 | downloadUrlPaths.length, 67 | ); 68 | 69 | const downloadUrls: string[] = []; 70 | 71 | // check if it has three levels 72 | for (let i = 0; i < flatDownloadUrls.length; i += 1) { 73 | if (flatDownloadUrls[i].download_url === null) { 74 | // eslint-disable-next-line no-await-in-loop 75 | const url: string = await handleThreeLevelDeep(flatDownloadUrls[i].path); 76 | downloadUrls.push(url); 77 | } else { 78 | downloadUrls.push(flatDownloadUrls[i].download_url); 79 | } 80 | } 81 | 82 | return downloadUrls; 83 | }; 84 | 85 | const getPackageYamls = async (manifests: string[]): Promise => { 86 | const downloadUrls = await (await getPackageDownloadUrls(manifests)).filter(x => x != null); 87 | 88 | const packageYamls = Promise.all( 89 | downloadUrls.map((url) => fetch(url, { 90 | headers: { 91 | Authorization: `token ${GITHUB_TOKEN}`, 92 | }, 93 | }) 94 | .then(res => res.buffer()) 95 | .then(buffer => { 96 | const utf8decoder = new TextDecoder("utf-8"); 97 | const utf16decoder = new TextDecoder("utf-16"); 98 | 99 | let res; 100 | 101 | try { 102 | const text = utf8decoder.decode(buffer); 103 | res = jsYaml.safeLoad(text); 104 | } catch (error) { 105 | const text = utf16decoder.decode(buffer); 106 | res = jsYaml.safeLoad(text); 107 | } 108 | 109 | res.Version = String(res.Version); 110 | 111 | return res; 112 | })), 113 | ); 114 | 115 | return packageYamls; 116 | }; 117 | 118 | export = { 119 | getPackageYamls, 120 | }; 121 | -------------------------------------------------------------------------------- /src/server/ghService/import/singlePackageImport.ts: -------------------------------------------------------------------------------- 1 | import fetch from "node-fetch"; 2 | import { TextDecoder } from "util"; 3 | import * as jsYaml from "js-yaml"; 4 | 5 | import { ManifestFolderList } from "../types/import/manifestFolderListModel"; 6 | 7 | const { 8 | GITHUB_TOKEN, 9 | } = process.env; 10 | 11 | const BASE_URL = "https://api.github.com/repositories/197275551/contents"; 12 | 13 | const getPackageDownloadUrl = async (manifestPath: string): Promise => { 14 | const downloadUrlPath: Promise = await fetch(`${BASE_URL}/${manifestPath}`, { 15 | headers: { 16 | Authorization: `token ${GITHUB_TOKEN}`, 17 | }, 18 | }).then(res => res.json()); 19 | 20 | const downloadUrl = (await downloadUrlPath).download_url; 21 | 22 | return downloadUrl; 23 | }; 24 | 25 | const getPackageYaml = async (manifestPath: string): Promise => { 26 | const downloadUrl = await getPackageDownloadUrl(manifestPath); 27 | 28 | const packageYaml = await fetch(downloadUrl) 29 | .then(res => res.buffer()) 30 | .then(buffer => { 31 | const utf8decoder = new TextDecoder("utf-8"); 32 | const utf16decoder = new TextDecoder("utf-16"); 33 | 34 | let res; 35 | 36 | try { 37 | const text = utf8decoder.decode(buffer); 38 | res = jsYaml.safeLoad(text); 39 | } catch (error) { 40 | const text = utf16decoder.decode(buffer); 41 | res = jsYaml.safeLoad(text); 42 | } 43 | 44 | res.Version = String(res.Version); 45 | 46 | return res; 47 | }); 48 | 49 | return packageYaml; 50 | }; 51 | 52 | export = { 53 | getPackageYaml, 54 | }; 55 | -------------------------------------------------------------------------------- /src/server/ghService/index.ts: -------------------------------------------------------------------------------- 1 | import importPackageUtil from "./import/importPackageUtil"; 2 | import manualPackageImportUtil from "./import/manualImportUtil"; 3 | import updatePackageUtil from "./update/updatePackageUtil"; 4 | import manualPackageUpdateUtil from "./update/manualPackageUpdateUtil"; 5 | import singlePackageImportUtil from "./import/singlePackageImport"; 6 | 7 | const initialPackageImport = async (): Promise => { 8 | const packageYamls = await importPackageUtil.getPackageYamls(); 9 | 10 | return packageYamls; 11 | }; 12 | 13 | const manualPackageImport = async (manifests: string[]): Promise => { 14 | const importedYaml = await manualPackageImportUtil.getPackageYamls(manifests); 15 | return importedYaml; 16 | }; 17 | 18 | const updatePackages = async (): Promise => { 19 | const updatePackageYamls = await updatePackageUtil.getUpdatedPackageYamls(); 20 | 21 | return updatePackageYamls; 22 | }; 23 | 24 | const manualPackageUpdate = async (since: Date, until: Date): Promise => { 25 | const updatedPackageYamls = await manualPackageUpdateUtil.getUpdatedPackageYamls(since, until); 26 | 27 | return updatedPackageYamls; 28 | }; 29 | 30 | const importSinglePackage = async (manifestPath: string): Promise => { 31 | const singleYaml = await singlePackageImportUtil.getPackageYaml(manifestPath); 32 | 33 | return singleYaml; 34 | }; 35 | 36 | export = { 37 | initialPackageImport, 38 | manualPackageImport, 39 | updatePackages, 40 | manualPackageUpdate, 41 | importSinglePackage, 42 | }; 43 | -------------------------------------------------------------------------------- /src/server/ghService/types/import/batchImportModel.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable camelcase */ 2 | 3 | export interface BatchImport { 4 | total_count: number; 5 | incomplete_results: boolean; 6 | items: Item[]; 7 | } 8 | 9 | export interface Item { 10 | name: string; 11 | path: string; 12 | sha: string; 13 | url: string; 14 | git_url: string; 15 | html_url: string; 16 | repository: Repository; 17 | score: number; 18 | } 19 | 20 | export interface Repository { 21 | id: number; 22 | node_id: string; 23 | name: string; 24 | full_name: string; 25 | private: boolean; 26 | owner: Owner; 27 | html_url: string; 28 | description: string; 29 | fork: boolean; 30 | url: string; 31 | forks_url: string; 32 | keys_url: string; 33 | collaborators_url: string; 34 | teams_url: string; 35 | hooks_url: string; 36 | issue_events_url: string; 37 | events_url: string; 38 | assignees_url: string; 39 | branches_url: string; 40 | tags_url: string; 41 | blobs_url: string; 42 | git_tags_url: string; 43 | git_refs_url: string; 44 | trees_url: string; 45 | statuses_url: string; 46 | languages_url: string; 47 | stargazers_url: string; 48 | contributors_url: string; 49 | subscribers_url: string; 50 | subscription_url: string; 51 | commits_url: string; 52 | git_commits_url: string; 53 | comments_url: string; 54 | issue_comment_url: string; 55 | contents_url: string; 56 | compare_url: string; 57 | merges_url: string; 58 | archive_url: string; 59 | downloads_url: string; 60 | issues_url: string; 61 | pulls_url: string; 62 | milestones_url: string; 63 | notifications_url: string; 64 | labels_url: string; 65 | releases_url: string; 66 | deployments_url: string; 67 | } 68 | 69 | export interface Owner { 70 | login: string; 71 | id: number; 72 | node_id: string; 73 | avatar_url: string; 74 | gravatar_id: string; 75 | url: string; 76 | html_url: string; 77 | followers_url: string; 78 | following_url: string; 79 | gists_url: string; 80 | starred_url: string; 81 | subscriptions_url: string; 82 | organizations_url: string; 83 | repos_url: string; 84 | events_url: string; 85 | received_events_url: string; 86 | type: string; 87 | site_admin: boolean; 88 | } 89 | -------------------------------------------------------------------------------- /src/server/ghService/types/import/manifestFolderListModel.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable camelcase */ 2 | // Generated by https://quicktype.io 3 | 4 | export interface ManifestFolderList { 5 | name: string; 6 | path: string; 7 | sha: string; 8 | size: number; 9 | url: string; 10 | html_url: string; 11 | git_url: string; 12 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 13 | download_url?: any; 14 | type: string; 15 | _links: Links; 16 | } 17 | 18 | interface Links { 19 | self: string; 20 | git: string; 21 | html: string; 22 | } 23 | -------------------------------------------------------------------------------- /src/server/ghService/types/update/commitDetailsModel.ts: -------------------------------------------------------------------------------- 1 | // Generated by https://quicktype.io 2 | 3 | /* eslint-disable camelcase */ 4 | export interface CommitDetails { 5 | sha: string; 6 | node_id: string; 7 | commit: Commit; 8 | url: string; 9 | html_url: string; 10 | comments_url: string; 11 | author: CommitDetailsAuthor; 12 | committer: CommitDetailsAuthor; 13 | parents: Parent[]; 14 | stats: Stats; 15 | files: File[]; 16 | } 17 | 18 | interface CommitDetailsAuthor { 19 | login: string; 20 | id: number; 21 | node_id: string; 22 | avatar_url: string; 23 | gravatar_id: string; 24 | url: string; 25 | html_url: string; 26 | followers_url: string; 27 | following_url: string; 28 | gists_url: string; 29 | starred_url: string; 30 | subscriptions_url: string; 31 | organizations_url: string; 32 | repos_url: string; 33 | events_url: string; 34 | received_events_url: string; 35 | type: string; 36 | site_admin: boolean; 37 | } 38 | 39 | interface Commit { 40 | author: CommitAuthor; 41 | committer: CommitAuthor; 42 | message: string; 43 | tree: Tree; 44 | url: string; 45 | comment_count: number; 46 | verification: Verification; 47 | } 48 | 49 | interface CommitAuthor { 50 | name: string; 51 | email: string; 52 | date: string; 53 | } 54 | 55 | interface Tree { 56 | sha: string; 57 | url: string; 58 | } 59 | 60 | interface Verification { 61 | verified: boolean; 62 | reason: string; 63 | signature: string; 64 | payload: string; 65 | } 66 | 67 | export interface File { 68 | sha: string; 69 | filename: string; 70 | status: string; 71 | additions: number; 72 | deletions: number; 73 | changes: number; 74 | blob_url: string; 75 | raw_url: string; 76 | contents_url: string; 77 | patch: string; 78 | } 79 | 80 | interface Parent { 81 | sha: string; 82 | url: string; 83 | html_url: string; 84 | } 85 | 86 | interface Stats { 87 | total: number; 88 | additions: number; 89 | deletions: number; 90 | } 91 | -------------------------------------------------------------------------------- /src/server/ghService/types/update/masterCommitModel.ts: -------------------------------------------------------------------------------- 1 | // Generated by https://quicktype.io 2 | 3 | /* eslint-disable camelcase */ 4 | export interface MasterCommit { 5 | sha: string; 6 | node_id: string; 7 | commit: Commit; 8 | url: string; 9 | html_url: string; 10 | comments_url: string; 11 | author: MasterCommitAuthor; 12 | committer: MasterCommitAuthor; 13 | parents: Parent[]; 14 | } 15 | 16 | interface MasterCommitAuthor { 17 | login: string; 18 | id: number; 19 | node_id: string; 20 | avatar_url: string; 21 | gravatar_id: string; 22 | url: string; 23 | html_url: string; 24 | followers_url: string; 25 | following_url: string; 26 | gists_url: string; 27 | starred_url: string; 28 | subscriptions_url: string; 29 | organizations_url: string; 30 | repos_url: string; 31 | events_url: string; 32 | received_events_url: string; 33 | type: string; 34 | site_admin: boolean; 35 | } 36 | 37 | interface Commit { 38 | author: CommitAuthor; 39 | committer: CommitAuthor; 40 | message: string; 41 | tree: Tree; 42 | url: string; 43 | comment_count: number; 44 | verification: Verification; 45 | } 46 | 47 | interface CommitAuthor { 48 | name: string; 49 | email: string; 50 | date: string; 51 | } 52 | 53 | interface Tree { 54 | sha: string; 55 | url: string; 56 | } 57 | 58 | interface Verification { 59 | verified: boolean; 60 | reason: string; 61 | signature: string; 62 | payload: string; 63 | } 64 | 65 | interface Parent { 66 | sha: string; 67 | url: string; 68 | html_url: string; 69 | } 70 | -------------------------------------------------------------------------------- /src/server/ghService/types/update/packageFileDetailsModel.ts: -------------------------------------------------------------------------------- 1 | // Generated by https://quicktype.io 2 | 3 | /* eslint-disable camelcase */ 4 | export interface PackageFileDetails { 5 | name: string; 6 | path: string; 7 | sha: string; 8 | size: number; 9 | url: string; 10 | html_url: string; 11 | git_url: string; 12 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 13 | download_url?: any; 14 | type: string; 15 | _links: Links; 16 | } 17 | 18 | export interface Links { 19 | self: string; 20 | git: string; 21 | html: string; 22 | } 23 | -------------------------------------------------------------------------------- /src/server/ghService/update/manualPackageUpdateUtil.ts: -------------------------------------------------------------------------------- 1 | import fetch from "node-fetch"; 2 | import * as jsYaml from "js-yaml"; 3 | import { TextDecoder } from "util"; 4 | 5 | import { MasterCommit } from "../types/update/masterCommitModel"; 6 | import { CommitDetails, File } from "../types/update/commitDetailsModel"; 7 | import { PackageFileDetails } from "../types/update/packageFileDetailsModel"; 8 | 9 | 10 | const COMMITS_BASE_URL = "https://api.github.com/repos/microsoft/winget-pkgs/commits?ref=master"; 11 | const CONTENTS_BASE_URL = "https://api.github.com/repos/microsoft/winget-pkgs/contents"; 12 | 13 | const { 14 | GITHUB_TOKEN, 15 | } = process.env; 16 | 17 | const getCommitsMasterTimeRange = async (since: Date, until: Date): Promise => { 18 | const masterCommits: Promise = await fetch( 19 | `${COMMITS_BASE_URL}&&since=${since}&&until=${until}`, 20 | { 21 | headers: { 22 | Authorization: `token ${GITHUB_TOKEN}`, 23 | }, 24 | }, 25 | ).then((res) => res.json()); 26 | 27 | const commitUrls = (await masterCommits).map((commit) => commit.url); 28 | 29 | return commitUrls; 30 | }; 31 | 32 | const getUpdatedFileFath = async (since: Date, until: Date): Promise => { 33 | const commitUrls = await getCommitsMasterTimeRange(since, until); 34 | 35 | const commitDetails: CommitDetails[] = await Promise.all( 36 | commitUrls.map((commitUrl) => fetch(commitUrl, { 37 | headers: { 38 | Authorization: `token ${GITHUB_TOKEN}`, 39 | }, 40 | }).then((res) => res.json())), 41 | ); 42 | 43 | const files = commitDetails.map((commitDetail) => commitDetail.files); 44 | // eslint-disable-next-line @typescript-eslint/ban-ts-ignore 45 | // @ts-ignore 46 | const flatFiles: File[] = files.flat(files.length); 47 | const filePaths = flatFiles.map((file) => file.filename); 48 | 49 | return filePaths; 50 | }; 51 | 52 | const getPackageDownloadUrls = async (since: Date, until: Date): Promise => { 53 | const updatedFilePaths = await (await getUpdatedFileFath(since, until)).filter((x) => x.startsWith("manifests/")); 54 | 55 | const packageFileDetails: PackageFileDetails[] = await Promise.all( 56 | updatedFilePaths.map((path) => fetch(`${CONTENTS_BASE_URL}/${path}`, { 57 | headers: { 58 | Authorization: `token ${GITHUB_TOKEN}`, 59 | }, 60 | }).then((res) => res.json())), 61 | ); 62 | 63 | const downloadUrls = packageFileDetails.map((pkg) => pkg.download_url); 64 | 65 | return downloadUrls; 66 | }; 67 | 68 | const getUpdatedPackageYamls = async (since: Date, until: Date): Promise => { 69 | const downloadUrls = await (await getPackageDownloadUrls(since, until)).filter( 70 | (url) => url != null, 71 | ); 72 | 73 | const updatePackageYamls = Promise.all( 74 | downloadUrls.map((url) => fetch(url, { 75 | headers: { 76 | Authorization: `token ${GITHUB_TOKEN}`, 77 | }, 78 | }) 79 | .then(res => res.buffer()) 80 | .then(buffer => { 81 | const utf8decoder = new TextDecoder("utf-8"); 82 | const utf16decoder = new TextDecoder("utf-16"); 83 | 84 | let res; 85 | 86 | try { 87 | const text = utf8decoder.decode(buffer); 88 | res = jsYaml.safeLoad(text); 89 | } catch (error) { 90 | const text = utf16decoder.decode(buffer); 91 | res = jsYaml.safeLoad(text); 92 | } 93 | 94 | res.Version = String(res.Version); 95 | 96 | return res; 97 | })), 98 | ); 99 | 100 | return updatePackageYamls; 101 | }; 102 | 103 | export = { getUpdatedPackageYamls }; 104 | -------------------------------------------------------------------------------- /src/server/ghService/update/updatePackageUtil.ts: -------------------------------------------------------------------------------- 1 | import fetch from "node-fetch"; 2 | import * as jsYaml from "js-yaml"; 3 | import { TextDecoder } from "util"; 4 | 5 | import { MasterCommit } from "../types/update/masterCommitModel"; 6 | import { CommitDetails, File } from "../types/update/commitDetailsModel"; 7 | import { PackageFileDetails } from "../types/update/packageFileDetailsModel"; 8 | 9 | 10 | const COMMITS_BASE_URL = "https://api.github.com/repos/microsoft/winget-pkgs/commits?ref=master"; 11 | const CONTENTS_BASE_URL = "https://api.github.com/repos/microsoft/winget-pkgs/contents"; 12 | 13 | const { 14 | GITHUB_TOKEN, 15 | UPDATE_FREQUENCY_MINUTES, 16 | } = process.env; 17 | 18 | const getCommitsMasterTimeRange = async (): Promise => { 19 | // eslint-disable-next-line radix 20 | const frequency = parseInt(UPDATE_FREQUENCY_MINUTES); 21 | 22 | // ! use when in production 23 | const since = new Date(new Date().setMinutes(new Date().getMinutes() - frequency)).toISOString(); 24 | const until = new Date().toISOString(); 25 | 26 | 27 | const masterCommits: Promise = await fetch( 28 | `${COMMITS_BASE_URL}&&since=${since}&&until=${until}`, 29 | { 30 | headers: { 31 | Authorization: `token ${GITHUB_TOKEN}`, 32 | }, 33 | }, 34 | ).then((res) => res.json()); 35 | 36 | const commitUrls = (await masterCommits).map((commit) => commit.url); 37 | 38 | return commitUrls; 39 | }; 40 | 41 | const getUpdatedFileFath = async (): Promise => { 42 | const commitUrls = await getCommitsMasterTimeRange(); 43 | 44 | const commitDetails: CommitDetails[] = await Promise.all( 45 | commitUrls.map((commitUrl) => fetch(commitUrl, { 46 | headers: { 47 | Authorization: `token ${GITHUB_TOKEN}`, 48 | }, 49 | }).then((res) => res.json())), 50 | ); 51 | 52 | const files = commitDetails.map((commitDetail) => commitDetail.files); 53 | // eslint-disable-next-line @typescript-eslint/ban-ts-ignore 54 | // @ts-ignore 55 | const flatFiles: File[] = files.flat(files.length); 56 | const filePaths = flatFiles.map((file) => file.filename); 57 | 58 | return filePaths; 59 | }; 60 | 61 | const getPackageDownloadUrls = async (): Promise => { 62 | const updatedFilePaths = await (await getUpdatedFileFath()).filter((x) => x.startsWith("manifests/")); 63 | 64 | const packageFileDetails: PackageFileDetails[] = await Promise.all( 65 | updatedFilePaths.map((path) => fetch(`${CONTENTS_BASE_URL}/${path}`, { 66 | headers: { 67 | Authorization: `token ${GITHUB_TOKEN}`, 68 | }, 69 | }).then((res) => res.json())), 70 | ); 71 | 72 | const downloadUrls = packageFileDetails.map((pkg) => pkg.download_url); 73 | 74 | return downloadUrls; 75 | }; 76 | 77 | const getUpdatedPackageYamls = async (): Promise => { 78 | const downloadUrls = await (await getPackageDownloadUrls()).filter( 79 | (url) => url != null, 80 | ); 81 | 82 | const updatePackageYamls = Promise.all( 83 | downloadUrls.map((url) => fetch(url, { 84 | headers: { 85 | Authorization: `token ${GITHUB_TOKEN}`, 86 | }, 87 | }) 88 | .then(res => res.buffer()) 89 | .then(buffer => { 90 | const utf8decoder = new TextDecoder("utf-8"); 91 | const utf16decoder = new TextDecoder("utf-16"); 92 | 93 | let res; 94 | 95 | try { 96 | const text = utf8decoder.decode(buffer); 97 | res = jsYaml.safeLoad(text); 98 | } catch (error) { 99 | const text = utf16decoder.decode(buffer); 100 | res = jsYaml.safeLoad(text); 101 | } 102 | 103 | res.Version = String(res.Version); 104 | 105 | return res; 106 | })), 107 | ); 108 | 109 | return updatePackageYamls; 110 | }; 111 | 112 | export = { getUpdatedPackageYamls }; 113 | -------------------------------------------------------------------------------- /src/server/helpers/index.ts: -------------------------------------------------------------------------------- 1 | import validateApiToken from "./validateApiToken"; 2 | 3 | export { 4 | // eslint-disable-next-line import/prefer-default-export 5 | validateApiToken, 6 | }; 7 | -------------------------------------------------------------------------------- /src/server/helpers/validateApiToken.ts: -------------------------------------------------------------------------------- 1 | import { ServerResponse } from "http"; 2 | 3 | import { FastifyRequest, FastifyReply } from "fastify"; 4 | 5 | const { 6 | API_ACCESS_TOKEN, 7 | } = process.env; 8 | 9 | const validateApiToken = async (request: FastifyRequest, reply: FastifyReply): Promise => { 10 | const accessToken = request.headers["xxx-access-token"]; 11 | if (accessToken == null) { 12 | reply.status(401); 13 | throw new Error("unauthorised"); 14 | } 15 | if (accessToken !== API_ACCESS_TOKEN) { 16 | reply.status(403); 17 | throw new Error("forbidden"); 18 | } 19 | }; 20 | 21 | export default validateApiToken; 22 | -------------------------------------------------------------------------------- /src/server/index.ts: -------------------------------------------------------------------------------- 1 | import buildFastify from "./buildFastify"; 2 | 3 | const { 4 | WEBSERVER_LOGGER, 5 | WEBSERVER_PORT, 6 | WEBSERVER_ADDRESS, 7 | } = process.env; 8 | 9 | const startServer = async (): Promise => { 10 | const server = buildFastify({ 11 | logger: WEBSERVER_LOGGER, 12 | }); 13 | 14 | try { 15 | await server.listen(Number.parseInt(WEBSERVER_PORT, 10), WEBSERVER_ADDRESS); 16 | server.log.info(`winget magic happens on port ${WEBSERVER_PORT}`); 17 | } catch (error) { 18 | server.log.error(error); 19 | process.exit(-1); 20 | } 21 | }; 22 | 23 | export { 24 | // eslint-disable-next-line import/prefer-default-export 25 | startServer, 26 | }; 27 | -------------------------------------------------------------------------------- /src/server/plugins/index.ts: -------------------------------------------------------------------------------- 1 | import ratelimit from "./ratelimit"; 2 | 3 | export { 4 | // eslint-disable-next-line import/prefer-default-export 5 | ratelimit, 6 | }; 7 | -------------------------------------------------------------------------------- /src/server/plugins/ratelimit.ts: -------------------------------------------------------------------------------- 1 | import { FastifyInstance } from "fastify"; 2 | 3 | // TODO: implement 4 | export default async (fastify: FastifyInstance): Promise => { 5 | fastify.addHook("onRequest", async () => { 6 | console.log("ratelimit plugin called"); 7 | }); 8 | }; 9 | -------------------------------------------------------------------------------- /src/server/routes/index.ts: -------------------------------------------------------------------------------- 1 | import { FastifyInstance } from "fastify"; 2 | 3 | import { PackageService } from "../../database"; 4 | 5 | import v1 from "./v1"; 6 | import v2 from "./v2"; 7 | 8 | export default async (fastify: FastifyInstance): Promise => { 9 | const packageService = new PackageService(); 10 | 11 | fastify.get("/", async () => ({ 12 | nonce: "cunty mcjim", 13 | rawrxd: "rawrxd", 14 | })); 15 | 16 | fastify.get("/healthz", async (request, reply) => { 17 | // test if the database conn is still ok (we dont currently have alerting for db errors) 18 | await packageService.findOne({ 19 | filters: {}, 20 | }); 21 | 22 | reply.status(204); 23 | }); 24 | 25 | fastify.register(v1, { prefix: "v1" }); 26 | fastify.register(v2, { prefix: "v2" }); 27 | }; 28 | -------------------------------------------------------------------------------- /src/server/routes/v1/index.ts: -------------------------------------------------------------------------------- 1 | import { FastifyInstance } from "fastify"; 2 | 3 | import { ratelimit } from "../../plugins"; 4 | import ghService from "../../ghService/index"; 5 | import { SortOrder } from "../../../database/types"; 6 | import { validateApiToken } from "../../helpers"; 7 | import { 8 | ManifestService, 9 | ManifestModel, 10 | addOrUpdatePackage, 11 | rebuildPackage, 12 | PackageService, 13 | } from "../../../database"; 14 | 15 | // NOTE: spec: https://github.com/microsoft/winget-cli/blob/master/doc/ManifestSpecv0.1.md 16 | // were more or less following it lel 17 | 18 | const MIN_PAGE_SIZE = 1; 19 | const MAX_PAGE_SIZE = 24; 20 | const DEFAULT_PAGE_SIZE = 12; 21 | 22 | const DEFAULT_AUTOCOMPLETE_SIZE = 3; 23 | 24 | const { 25 | NODE_ENV, 26 | } = process.env; 27 | 28 | // TODO: split this file up 29 | const autocompleteSchema = { 30 | querystring: { 31 | type: "object", 32 | required: ["query"], 33 | properties: { 34 | query: { 35 | type: "string", 36 | }, 37 | }, 38 | }, 39 | }; 40 | 41 | const searchSchema = { 42 | querystring: { 43 | type: "object", 44 | required: ["name"], 45 | properties: { 46 | name: { 47 | type: "string", 48 | }, 49 | sort: { 50 | type: "string", 51 | // TODO: make every field in PackageModel (give or take a few) sortable 52 | enum: ["Name", "updatedAt"], 53 | }, 54 | order: { 55 | type: "number", 56 | enum: Object.values(SortOrder), 57 | }, 58 | limit: { 59 | type: "number", 60 | nullable: true, 61 | minimum: MIN_PAGE_SIZE, 62 | maximum: MAX_PAGE_SIZE, 63 | }, 64 | page: { 65 | type: "number", 66 | nullable: true, 67 | minimum: 0, 68 | }, 69 | }, 70 | }, 71 | }; 72 | 73 | const orgSchema = { 74 | params: { 75 | type: "object", 76 | required: ["org"], 77 | properties: { 78 | org: { 79 | type: "string", 80 | minLength: 1, 81 | }, 82 | }, 83 | }, 84 | querystring: { 85 | type: "object", 86 | properties: { 87 | limit: { 88 | type: "number", 89 | nullable: true, 90 | minimum: MIN_PAGE_SIZE, 91 | maximum: MAX_PAGE_SIZE, 92 | }, 93 | page: { 94 | type: "number", 95 | nullable: true, 96 | minimum: 0, 97 | }, 98 | }, 99 | }, 100 | }; 101 | 102 | const orgPkgSchema = { 103 | params: { 104 | type: "object", 105 | required: ["org", "pkg"], 106 | properties: { 107 | org: { 108 | type: "string", 109 | minLength: 1, 110 | }, 111 | pkg: { 112 | type: "string", 113 | minLength: 1, 114 | }, 115 | }, 116 | }, 117 | }; 118 | 119 | // TODO: make sure this schema is ok (required fields?) 120 | const manualPackageUpdateSchema = { 121 | querystring: { 122 | type: "object", 123 | properties: { 124 | since: { 125 | type: "string", 126 | }, 127 | sort: { 128 | until: "string", 129 | }, 130 | }, 131 | }, 132 | }; 133 | 134 | // TODO: move this somewhere else 135 | enum ApiErrorType { 136 | VALIDATION_ERROR = "validation_error", 137 | GENERIC_CLIENT_ERROR = "generic_client_error", 138 | GENERIC_SERVER_ERROR = "generic_server_error", 139 | } 140 | 141 | interface IApiErrorResponse { 142 | error: { 143 | type: ApiErrorType; 144 | debug?: string; 145 | stack?: string; 146 | }; 147 | } 148 | 149 | export default async (fastify: FastifyInstance): Promise => { 150 | fastify.setErrorHandler(async (error, request, reply): Promise => { 151 | const [debug, stack] = NODE_ENV === "dev" ? [error.message, error.stack] : []; 152 | 153 | if (error.validation != null) { 154 | return { 155 | error: { 156 | type: ApiErrorType.VALIDATION_ERROR, 157 | debug, 158 | stack, 159 | }, 160 | }; 161 | } 162 | 163 | if (reply.res.statusCode.toString().startsWith("4")) { 164 | return { 165 | error: { 166 | type: ApiErrorType.GENERIC_CLIENT_ERROR, 167 | debug, 168 | stack, 169 | }, 170 | }; 171 | } 172 | 173 | return { 174 | error: { 175 | type: ApiErrorType.GENERIC_SERVER_ERROR, 176 | debug, 177 | stack, 178 | }, 179 | }; 180 | }); 181 | 182 | fastify.register(ratelimit, { 183 | nonce: "yes", 184 | }); 185 | 186 | // TODO: were cheking the header shit in a filthy way rn, make an auth middleware or something 187 | // *----------------- import package update -------------------- 188 | fastify.get("/ghs/import", { onRequest: validateApiToken }, async () => { 189 | const yamls = await ghService.initialPackageImport(); 190 | 191 | await Promise.all( 192 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 193 | yamls.map(yaml => addOrUpdatePackage(yaml as any)), 194 | ); 195 | 196 | return `imported ${yamls.length} packages at ${new Date().toISOString()}`; 197 | }); 198 | 199 | // TODO: same as /ghs/import 200 | // *----------------- update package update -------------------- 201 | fastify.get("/ghs/update", { onRequest: validateApiToken }, async () => { 202 | const updateYamls = await ghService.updatePackages(); 203 | 204 | if (updateYamls.length > 0) { 205 | for (let i = 0; i < updateYamls.length; i += 1) { 206 | const pkg = updateYamls[i] as unknown as ManifestModel; 207 | 208 | // eslint-disable-next-line no-await-in-loop 209 | await addOrUpdatePackage(pkg); 210 | } 211 | } 212 | 213 | return `${updateYamls.length} updated at ${new Date().toISOString()}`; 214 | }); 215 | 216 | // *----------------- manual package import--------------------- 217 | fastify.post("/ghs/manualImport", { onRequest: validateApiToken }, async request => { 218 | const manifests = request.body.manifests as string[]; 219 | 220 | const yamls = await ghService.manualPackageImport(manifests); 221 | 222 | await Promise.all( 223 | yamls.map(yaml => { 224 | const pkg = yaml as unknown as ManifestModel; 225 | 226 | return addOrUpdatePackage(pkg); 227 | }), 228 | ); 229 | 230 | return `imported ${yamls.length} packages at ${new Date().toISOString()}`; 231 | }); 232 | 233 | // *----------------- manual package update--------------------- 234 | fastify.get("/ghs/manualUpdate", { schema: manualPackageUpdateSchema, onRequest: validateApiToken }, async request => { 235 | const { since, until } = request.query; 236 | const updatedYamls = await ghService.manualPackageUpdate(since, until); 237 | 238 | if (updatedYamls.length > 0) { 239 | for (let i = 0; i < updatedYamls.length; i += 1) { 240 | const pkg = updatedYamls[i] as unknown as ManifestModel; 241 | 242 | // eslint-disable-next-line no-await-in-loop 243 | await addOrUpdatePackage(pkg); 244 | } 245 | } 246 | 247 | return `updated ${updatedYamls.length} packages at ${new Date().toISOString()}`; 248 | }); 249 | 250 | // *----------------- single package import --------------------- 251 | fastify.post("/ghs/singleImport", { onRequest: validateApiToken }, async request => { 252 | const manifestPath = request.body.manifestPath as string; 253 | const yaml = await ghService.importSinglePackage(manifestPath); 254 | 255 | if (yaml == null || yaml === "") { 256 | return "error no yaml found"; 257 | } 258 | 259 | const pkg = yaml as unknown as ManifestModel; 260 | 261 | const result = { insertedCount: 1 }; 262 | await addOrUpdatePackage(pkg); 263 | 264 | return `insertted ${result.insertedCount} with ID - ${pkg.Id}`; 265 | }); 266 | 267 | // *----------------- override package image --------------------- 268 | fastify.post("/ghs/imageOverride", { onRequest: validateApiToken }, async request => { 269 | const { pkgId, iconUrl } = request.body; 270 | 271 | // not optimised but here we are (will fix later, i rly will) 272 | const result = { modifiedCount: 1 }; 273 | await rebuildPackage(pkgId, { 274 | IconUrl: iconUrl, 275 | }); 276 | 277 | return `updated ${result.modifiedCount} iconUrl at ${new Date().toISOString()} for ID - ${pkgId}`; 278 | }); 279 | 280 | // *----------------- auto complete --------------------- 281 | fastify.get("/autocomplete", { schema: autocompleteSchema }, async request => { 282 | const { query } = request.query; 283 | 284 | const manifestService = new ManifestService(); 285 | const packages = await manifestService.findAutocomplete(query, DEFAULT_AUTOCOMPLETE_SIZE); 286 | 287 | return { 288 | packages, 289 | }; 290 | }); 291 | 292 | // TODO: cache a search for everything response as its probs expensive af (optimise in some way anyway) 293 | // TODO: could also make a seperate route which optimises a list all packages type thing 294 | fastify.get("/search", { schema: searchSchema }, async request => { 295 | const { 296 | name, 297 | sort = "Name", 298 | order = SortOrder.ASCENDING, 299 | limit = DEFAULT_PAGE_SIZE, 300 | page = 0, 301 | } = request.query; 302 | 303 | const manifestService = new ManifestService(); 304 | const [packages, total] = await manifestService.findByName(name, limit, page, sort, order); 305 | 306 | return { 307 | packages, 308 | total, 309 | }; 310 | }); 311 | 312 | // TODO: make it so the filters field is not required 313 | fastify.get("/list", async () => { 314 | const packageService = new PackageService(); 315 | 316 | // TODO: cant deselect _id, maybe add that opt to the service (thats why theres a map at the end rn) 317 | // Note: keeping the date field called 'updatedAt' instead of UpdatedAt for backwards compat 318 | const list = (await packageService.find({ 319 | filters: {}, 320 | select: [ 321 | "Id", 322 | "UpdatedAt", 323 | ], 324 | })).map(e => ({ Id: e.Id, updatedAt: e.UpdatedAt })); 325 | 326 | return { 327 | list, 328 | }; 329 | }); 330 | 331 | fastify.get("/:org", { schema: orgSchema }, async request => { 332 | const { org } = request.params; 333 | const { page = 0, limit = DEFAULT_PAGE_SIZE } = request.query; 334 | 335 | const manifestService = new ManifestService(); 336 | const [packages, total] = await manifestService.findByOrg(org, limit, page); 337 | 338 | return { 339 | packages, 340 | total, 341 | }; 342 | }); 343 | 344 | fastify.get("/:org/:pkg", { schema: orgPkgSchema }, async request => { 345 | const { org, pkg } = request.params; 346 | 347 | const manifestService = new ManifestService(); 348 | const orgPkg = await manifestService.findByPackage(org, pkg); 349 | 350 | return { 351 | package: orgPkg, 352 | }; 353 | }); 354 | }; 355 | -------------------------------------------------------------------------------- /src/server/routes/v2/featuredPackages.ts: -------------------------------------------------------------------------------- 1 | import { FastifyInstance } from "fastify"; 2 | 3 | import { PackageService } from "../../../database"; 4 | import { validateApiToken } from "../../helpers"; 5 | 6 | const updateFeaturedSchema = { 7 | params: { 8 | type: "object", 9 | required: ["id"], 10 | properties: { 11 | id: { 12 | type: "string", 13 | minLength: 3, 14 | }, 15 | }, 16 | }, 17 | body: { 18 | type: "object", 19 | required: ["Banner", "Logo"], 20 | properties: { 21 | Banner: { 22 | type: "string", 23 | minLength: 5, 24 | }, 25 | Logo: { 26 | type: "string", 27 | minLength: 5, 28 | }, 29 | }, 30 | }, 31 | }; 32 | 33 | const deleteFeaturedSchema = { 34 | params: { 35 | type: "object", 36 | required: ["id"], 37 | properties: { 38 | id: { 39 | type: "string", 40 | minLength: 3, 41 | }, 42 | }, 43 | }, 44 | }; 45 | 46 | export default async (fastify: FastifyInstance): Promise => { 47 | const packageService = new PackageService(); 48 | 49 | //* get featured packages 50 | fastify.get("/", async () => { 51 | const featuredPackages = await packageService.find({ 52 | filters: { Featured: true }, 53 | }); 54 | 55 | return { 56 | Packages: featuredPackages, 57 | }; 58 | }); 59 | 60 | //* update featured package 61 | fastify.post("/:id", { schema: updateFeaturedSchema, onRequest: validateApiToken }, async (req, res) => { 62 | const { id } = req.params; 63 | const { Banner, Logo } = req.body; 64 | const pkg = await packageService.findOne({ filters: { Id: id } }); 65 | 66 | if (pkg != null) { 67 | pkg.Featured = true; 68 | pkg.Banner = Banner; 69 | pkg.Logo = Logo; 70 | 71 | await packageService.upsertPackage(pkg); 72 | return { 73 | Message: "updated featured package details", 74 | }; 75 | } 76 | 77 | res.code(404); 78 | throw new Error("failed to update featured package details, may not exist"); 79 | }); 80 | 81 | //* delete featured packaged (clear image, set to false) 82 | fastify.delete("/:id", { schema: deleteFeaturedSchema, onRequest: validateApiToken }, async (req, res) => { 83 | const { id } = req.params; 84 | const pkg = await packageService.findOne({ filters: { Id: id } }); 85 | 86 | if (pkg != null) { 87 | pkg.Featured = false; 88 | pkg.Banner = ""; 89 | pkg.Logo = ""; 90 | 91 | await packageService.upsertPackage(pkg); 92 | return { 93 | Message: "deleted featured package details", 94 | }; 95 | } 96 | 97 | res.code(404); 98 | throw new Error( 99 | "failed to delete featured package details, package may not exist", 100 | ); 101 | }); 102 | 103 | //* delete all featured packages 104 | fastify.delete("/", { onRequest: validateApiToken }, async (req, res) => { 105 | const removedPackages = await packageService.update({ 106 | filters: { Featured: true }, 107 | update: { Featured: false, Banner: "", Logo: "" }, 108 | }); 109 | 110 | if (removedPackages.modifiedCount === 0) { 111 | res.status(404); 112 | throw new Error("no featured packages found"); 113 | } 114 | 115 | return { 116 | Message: `Removed ${removedPackages.modifiedCount} featured packages`, 117 | }; 118 | }); 119 | }; 120 | -------------------------------------------------------------------------------- /src/server/routes/v2/github.ts: -------------------------------------------------------------------------------- 1 | import { FastifyInstance } from "fastify"; 2 | import { ManifestModel, addOrUpdatePackage } from "../../../database"; 3 | import { validateApiToken } from "../../helpers"; 4 | import ghService from "../../ghService/index"; 5 | 6 | export default async (fastify: FastifyInstance): Promise => { 7 | // *----------------- import package update -------------------- 8 | fastify.get("/import", { onRequest: validateApiToken }, async () => { 9 | const yamls = await ghService.initialPackageImport(); 10 | console.log(`yamls length - ${yamls.length}`); 11 | 12 | const batchSize = yamls.length % 2 === 0 ? 150 : 155; 13 | let batchIndex = 0; 14 | 15 | while (batchIndex < yamls.length) { 16 | const batch = yamls.slice(batchIndex, batchIndex + batchSize); 17 | 18 | // eslint-disable-next-line no-await-in-loop 19 | await Promise.all( 20 | // eslint-disable-next-line no-loop-func 21 | batch.map(async x => { 22 | const pkg = x as unknown as ManifestModel; 23 | console.log(`${pkg.Id} / ${pkg.Version} - ${batchIndex}`); 24 | 25 | await addOrUpdatePackage(pkg); 26 | }), 27 | ); 28 | 29 | batchIndex += batchSize; 30 | } 31 | 32 | return { 33 | Message: `imported ${ 34 | yamls.length 35 | } packages at ${new Date().toISOString()}`, 36 | }; 37 | }); 38 | 39 | // *----------------- manual package import--------------------- 40 | fastify.post( 41 | "/manualImport", 42 | { onRequest: validateApiToken }, 43 | async (req) => { 44 | const manifests = req.body.manifests as string[]; 45 | const yamls = await ghService.manualPackageImport(manifests); 46 | 47 | console.log(`yamls length - ${yamls.length}`); 48 | 49 | const BATCH_SIZE = yamls.length % 2 === 0 ? 4 : 5; 50 | let batchIndex = 0; 51 | 52 | while (batchIndex < yamls.length) { 53 | const batch = yamls.slice(batchIndex, batchIndex + BATCH_SIZE); 54 | 55 | // eslint-disable-next-line no-await-in-loop 56 | await Promise.all( 57 | // eslint-disable-next-line no-loop-func 58 | batch.map(async x => { 59 | const pkg = x as unknown as ManifestModel; 60 | console.log(`${pkg.Id} / ${pkg.Version} - ${batchIndex}`); 61 | 62 | await addOrUpdatePackage(pkg); 63 | }), 64 | ); 65 | 66 | batchIndex += BATCH_SIZE; 67 | } 68 | 69 | return { 70 | Message: `imported ${yamls.length} packages at ${new Date().toISOString()}`, 71 | }; 72 | }, 73 | ); 74 | }; 75 | -------------------------------------------------------------------------------- /src/server/routes/v2/index.ts: -------------------------------------------------------------------------------- 1 | import { FastifyInstance } from "fastify"; 2 | 3 | import stats from "./stats"; 4 | import packages from "./packages"; 5 | import manifests from "./manifests"; 6 | import featuredPackages from "./featuredPackages"; 7 | import github from "./github"; 8 | 9 | export default async (fastify: FastifyInstance): Promise => { 10 | fastify.register(stats, { prefix: "stats" }); 11 | fastify.register(packages, { prefix: "packages" }); 12 | fastify.register(manifests, { prefix: "manifests" }); 13 | fastify.register(featuredPackages, { prefix: "featured" }); 14 | fastify.register(github, { prefix: "ghs" }); 15 | }; 16 | -------------------------------------------------------------------------------- /src/server/routes/v2/manifests.ts: -------------------------------------------------------------------------------- 1 | import { FastifyInstance } from "fastify"; 2 | import { ManifestService } from "../../../database"; 3 | 4 | const manifestSchema = { 5 | params: { 6 | type: "object", 7 | required: ["id", "version"], 8 | properties: { 9 | id: { 10 | type: "string", 11 | // a.b 12 | minLength: 3, 13 | }, 14 | version: { 15 | type: "string", 16 | minLength: 1, 17 | }, 18 | }, 19 | }, 20 | }; 21 | 22 | export default async (fastify: FastifyInstance): Promise => { 23 | const manifestService = new ManifestService(); 24 | 25 | fastify.get("/:id/:version", { schema: manifestSchema }, async (request, response) => { 26 | const { id, version } = request.params; 27 | 28 | const manifest = await manifestService.findManifestVersion(id, version); 29 | if (manifest == null) { 30 | response.code(404); 31 | throw new Error("manifest not found"); 32 | } 33 | 34 | return { 35 | Manifest: manifest, 36 | }; 37 | }); 38 | }; 39 | -------------------------------------------------------------------------------- /src/server/routes/v2/packages.ts: -------------------------------------------------------------------------------- 1 | import { FastifyInstance } from "fastify"; 2 | import { PackageService, StatsService } from "../../../database"; 3 | import { PackageSortFields, SortOrder } from "../../../database/types"; 4 | 5 | const DEFAULT_PAGE = 0; 6 | const DEFAULT_PAGE_SIZE = 12; 7 | const MAX_PAGE_SIZE = 24; 8 | 9 | const packageSchema = { 10 | querystring: { 11 | type: "object", 12 | properties: { 13 | query: { 14 | type: "string", 15 | minLength: 1, 16 | }, 17 | name: { 18 | type: "string", 19 | minLength: 1, 20 | }, 21 | publisher: { 22 | type: "string", 23 | minLength: 1, 24 | }, 25 | description: { 26 | type: "string", 27 | minLength: 1, 28 | }, 29 | tags: { 30 | type: "string", 31 | minLength: 1, 32 | }, 33 | take: { 34 | type: "number", 35 | min: 1, 36 | max: MAX_PAGE_SIZE, 37 | }, 38 | page: { 39 | type: "number", 40 | min: 0, 41 | }, 42 | sort: { 43 | type: "string", 44 | enum: Object.values(PackageSortFields), 45 | }, 46 | order: { 47 | type: "number", 48 | enum: [...Object.values(SortOrder), "SearchScore"], 49 | }, 50 | splitQuery: { 51 | type: "boolean", 52 | }, 53 | partialMatch: { 54 | type: "boolean", 55 | }, 56 | ensureContains: { 57 | type: "boolean", 58 | }, 59 | }, 60 | }, 61 | }; 62 | 63 | const publisherPackageSchema = { 64 | params: { 65 | type: "object", 66 | required: ["publisher"], 67 | properties: { 68 | publisher: { 69 | type: "string", 70 | minLength: 1, 71 | }, 72 | }, 73 | }, 74 | querystring: { 75 | type: "object", 76 | properties: { 77 | take: { 78 | type: "number", 79 | min: 1, 80 | }, 81 | page: { 82 | type: "number", 83 | min: 0, 84 | }, 85 | sort: { 86 | type: "string", 87 | enum: Object.values(PackageSortFields), 88 | }, 89 | order: { 90 | type: "number", 91 | enum: Object.values(SortOrder), 92 | }, 93 | }, 94 | }, 95 | }; 96 | 97 | const singlePackageSchema = { 98 | params: { 99 | type: "object", 100 | required: ["publisher", "packageName"], 101 | properties: { 102 | publisher: { 103 | type: "string", 104 | minLength: 1, 105 | }, 106 | packageName: { 107 | type: "string", 108 | minLength: 1, 109 | }, 110 | }, 111 | }, 112 | }; 113 | 114 | export default async (fastify: FastifyInstance): Promise => { 115 | const packageService = new PackageService(); 116 | const statsService = new StatsService(); 117 | 118 | // NOTE: query searches name > publisher > description 119 | // NOTE: tags are exact match, separated by ',' 120 | fastify.get("/", { schema: packageSchema }, async request => { 121 | const { 122 | query, 123 | name, 124 | publisher, 125 | description, 126 | tags, 127 | take = DEFAULT_PAGE_SIZE, 128 | page = DEFAULT_PAGE, 129 | sort = "SearchScore", 130 | order = SortOrder.ASCENDING, 131 | splitQuery = true, 132 | partialMatch = false, 133 | ensureContains = false, 134 | } = request.query; 135 | 136 | // NOTE: fastify auto parses it as a boolean (yay!) 137 | const searchOptions = { 138 | splitQuery, 139 | partialMatch, 140 | ensureContains, 141 | }; 142 | 143 | const [pkgs, total] = await packageService.searchPackages({ 144 | query, 145 | name, 146 | publisher, 147 | description, 148 | ...(tags == null ? {} : { tags: tags.split(",") }), 149 | }, take, page, sort, order, searchOptions); 150 | 151 | return { 152 | Packages: pkgs, 153 | Total: total, 154 | }; 155 | }); 156 | 157 | fastify.get("/:publisher", { schema: publisherPackageSchema }, async request => { 158 | const { publisher } = request.params; 159 | const { 160 | take = DEFAULT_PAGE_SIZE, 161 | page = DEFAULT_PAGE, 162 | sort = PackageSortFields.LatestName, 163 | order = SortOrder.ASCENDING, 164 | } = request.query; 165 | 166 | const [pkgs, total] = await packageService.findByPublisher(publisher, take, page, sort, order); 167 | 168 | return { 169 | Packages: pkgs, 170 | Total: total, 171 | }; 172 | }); 173 | 174 | fastify.get("/:publisher/:packageName", { schema: singlePackageSchema }, async (request, response) => { 175 | const { publisher, packageName } = request.params; 176 | 177 | const pkg = await packageService.findSinglePackage(publisher, packageName); 178 | if (pkg == null) { 179 | response.code(404); 180 | throw new Error("package not found"); 181 | } 182 | 183 | statsService.incrementAccessCount(`${publisher}.${packageName}`); 184 | 185 | return { 186 | Package: pkg, 187 | }; 188 | }); 189 | }; 190 | -------------------------------------------------------------------------------- /src/server/routes/v2/stats.ts: -------------------------------------------------------------------------------- 1 | import { FastifyInstance } from "fastify"; 2 | import moment from "moment"; 3 | 4 | import { StatsService, StatsResolution } from "../../../database"; 5 | 6 | const dayInMs = 1000 * 60 * 60 * 24; 7 | 8 | const statsSchema = { 9 | querystring: { 10 | type: "object", 11 | required: ["packageId", "resolution", "after"], 12 | properties: { 13 | packageId: { 14 | type: "string", 15 | }, 16 | resolution: { 17 | type: "string", 18 | enum: Object.values(StatsResolution), 19 | }, 20 | after: { 21 | type: "string", 22 | }, 23 | before: { 24 | type: "string", 25 | nullable: true, 26 | }, 27 | }, 28 | }, 29 | }; 30 | 31 | export default async (fastify: FastifyInstance): Promise => { 32 | const statsService = new StatsService(); 33 | 34 | fastify.get("/", { schema: statsSchema }, async (request, response) => { 35 | const { 36 | packageId, 37 | resolution, 38 | after, 39 | before = (new Date()).toISOString(), 40 | } = request.query; 41 | 42 | // limitations, decided to put them here as they are api imposed limitations rather 43 | // that ones imposed by functionality (like some query param combinations in search) 44 | let validDate = false; 45 | 46 | const afterDate = moment(after).utc().startOf(resolution); 47 | const beforeDate = moment(before).utc().startOf(resolution); 48 | 49 | // calculation: selected timeframe in ms <= allowed max timeframe 50 | const timeframeMs = beforeDate.valueOf() - afterDate.valueOf(); 51 | 52 | switch (resolution) { 53 | case StatsResolution.Day: { 54 | validDate = timeframeMs <= dayInMs * 7 * 4; 55 | 56 | break; 57 | } 58 | case StatsResolution.Week: { 59 | validDate = timeframeMs <= dayInMs * 7 * 8; 60 | 61 | break; 62 | } 63 | case StatsResolution.Month: { 64 | validDate = timeframeMs <= dayInMs * 7 * 12; 65 | 66 | break; 67 | } 68 | // not required cos exhausitve switch but what do i know, right eslint? 69 | default: { 70 | response.code(500); 71 | throw new Error("were all fucked"); 72 | } 73 | } 74 | 75 | if (!validDate) { 76 | response.code(400); 77 | throw new Error("invalid dates set, out of bounds of allowed values"); 78 | } 79 | 80 | const stats = await statsService.getPackageStats(packageId, resolution, new Date(after), new Date(before)); 81 | 82 | return { 83 | Stats: { 84 | Id: packageId, 85 | Data: stats, 86 | }, 87 | }; 88 | }); 89 | }; 90 | -------------------------------------------------------------------------------- /tests/setup.ts: -------------------------------------------------------------------------------- 1 | import dotenv from "dotenv"; 2 | import sourceMapSupport from "source-map-support"; 3 | 4 | // set up env vars and source maps 5 | // this is done via the cli when running code through node (non-test stuff) 6 | dotenv.config(); 7 | sourceMapSupport.install(); 8 | -------------------------------------------------------------------------------- /tests/test.spec.ts: -------------------------------------------------------------------------------- 1 | describe("test", () => { 2 | it("should work", () => { 3 | expect(1 + 2).toEqual(3); 4 | }); 5 | }); 6 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | /* Basic Options */ 4 | // "incremental": true, /* Enable incremental compilation */ 5 | "target": "ES2015", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */ 6 | "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ 7 | "lib": ["ES2015", "ESNext"], /* Specify library files to be included in the compilation. */ 8 | "allowJs": false, /* Allow javascript files to be compiled. */ 9 | // "checkJs": true, /* Report errors in .js files. */ 10 | // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ 11 | // "declaration": true, /* Generates corresponding '.d.ts' file. */ 12 | // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ 13 | // "sourceMap": true, /* Generates corresponding '.map' file. */ 14 | // "outFile": "./", /* Concatenate and emit output to single file. */ 15 | "outDir": "./dist", /* Redirect output structure to the directory. */ 16 | "rootDir": "./src", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ 17 | // "composite": true, /* Enable project compilation */ 18 | // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ 19 | // "removeComments": true, /* Do not emit comments to output. */ 20 | // "noEmit": true, /* Do not emit outputs. */ 21 | // "importHelpers": true, /* Import emit helpers from 'tslib'. */ 22 | // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ 23 | // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ 24 | 25 | /* Strict Type-Checking Options */ 26 | "strict": true, /* Enable all strict type-checking options. */ 27 | // "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */ 28 | // "strictNullChecks": true, /* Enable strict null checks. */ 29 | // "strictFunctionTypes": true, /* Enable strict checking of function types. */ 30 | // "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */ 31 | // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ 32 | // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ 33 | // "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ 34 | 35 | /* Additional Checks */ 36 | // "noUnusedLocals": true, /* Report errors on unused locals. */ 37 | // "noUnusedParameters": true, /* Report errors on unused parameters. */ 38 | // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ 39 | // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ 40 | 41 | /* Module Resolution Options */ 42 | // "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ 43 | // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ 44 | // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ 45 | // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ 46 | // "typeRoots": [], /* List of folders to include type definitions from. */ 47 | "types": [ 48 | "node", 49 | "jest", 50 | "reflect-metadata" 51 | ], /* Type declaration files to be included in compilation. */ 52 | // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ 53 | "esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ 54 | // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ 55 | // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ 56 | 57 | /* Source Map Options */ 58 | // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ 59 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 60 | // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ 61 | // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ 62 | 63 | /* Experimental Options */ 64 | "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ 65 | "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ 66 | 67 | /* Advanced Options */ 68 | "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ 69 | }, 70 | "exclude": [ 71 | "node_modules", 72 | "tests", 73 | "**/*.spec.ts", 74 | "**/*.test.ts" 75 | ] 76 | } 77 | --------------------------------------------------------------------------------