├── .gitattributes ├── sdk ├── test │ └── typescript │ │ ├── .gitignore │ │ ├── package.json │ │ └── test.ts ├── src │ └── typescript │ │ ├── src │ │ └── index.ts │ │ ├── tsconfig.json │ │ ├── rollup.config.js │ │ └── package.json └── config.yaml ├── src ├── config │ ├── test.ts │ ├── index.ts │ ├── development.ts │ ├── production.ts │ ├── schema.ts │ └── configuration.ts ├── core │ ├── context │ │ ├── index.ts │ │ └── context.ts │ ├── logger │ │ ├── index.ts │ │ ├── nest-logger.ts │ │ ├── logger.ts │ │ └── winston-logger.ts │ ├── models │ │ ├── index.ts │ │ └── models-decorators.ts │ ├── telemetry │ │ ├── index.ts │ │ ├── noop-telemetry.ts │ │ ├── telemetry.ts │ │ ├── factory.ts │ │ └── statsd-telemetry.ts │ ├── repo-auth │ │ ├── index.ts │ │ ├── repo-auth-decorators.ts │ │ ├── repo-auth.test.ts │ │ └── repo-auth.ts │ ├── pagination │ │ ├── index.ts │ │ ├── pagination.ts │ │ ├── paginated-response.ts │ │ └── pagination-decorators.ts │ └── index.ts ├── models │ ├── index.ts │ └── repo-reference.ts ├── services │ ├── db │ │ ├── index.ts │ │ └── db.service.ts │ ├── fs │ │ ├── index.ts │ │ └── fs.service.ts │ ├── http │ │ ├── index.ts │ │ └── http.service.ts │ ├── repo │ │ ├── index.ts │ │ ├── local-repo │ │ │ ├── index.ts │ │ │ ├── local-repo.ts │ │ │ └── local-repo.test.ts │ │ ├── mutex │ │ │ ├── index.ts │ │ │ ├── state-mutex.ts │ │ │ ├── mutex.test.ts │ │ │ └── mutex.ts │ │ └── repo.service.ts │ ├── branch │ │ ├── index.ts │ │ ├── branch.service.ts │ │ └── branch.service.test.ts │ ├── commit │ │ ├── index.ts │ │ └── commit.service.ts │ ├── compare │ │ ├── index.ts │ │ └── compare.service.ts │ ├── content │ │ ├── index.ts │ │ └── content.service.ts │ ├── disk-usage │ │ ├── index.ts │ │ └── disk-usage.service.ts │ ├── repo-cleanup │ │ ├── index.ts │ │ ├── repo-cleanup.service.ts │ │ └── repo-cleanup.service.test.ts │ ├── repo-index │ │ ├── index.ts │ │ ├── repo-index.service.test.ts │ │ └── repo-index.service.ts │ ├── permission │ │ ├── cache │ │ │ ├── index.ts │ │ │ ├── permission-cache.service.ts │ │ │ └── permission-cache.service.test.ts │ │ ├── permissions.ts │ │ ├── index.ts │ │ ├── permission.service.ts │ │ └── permission.service.test.ts │ ├── app.service.ts │ └── index.ts ├── definitions.d.ts ├── middlewares │ ├── index.ts │ ├── context │ │ ├── context.middleware.ts │ │ └── context.middleware.test.ts │ └── logger-interceptor.ts ├── utils │ ├── index.ts │ ├── repo-utils.ts │ ├── secure-utils.ts │ ├── git-utils.ts │ └── misc-utils.ts ├── scripts │ ├── generate-swagger-specs.ts │ ├── swagger.test.ts │ └── swagger-generation.ts ├── dtos │ ├── git-dir-object-content.ts │ ├── git-commit-ref.ts │ ├── git-submodule-object-content.ts │ ├── git-branch.ts │ ├── git-file-object-without-content.ts │ ├── git-signature.ts │ ├── git-file-object-with-content.ts │ ├── index.ts │ ├── git-object-content.ts │ ├── git-commit.ts │ ├── git-contents.ts │ ├── git-tree.ts │ ├── git-diff.ts │ └── git-file-diff.ts ├── controllers │ ├── index.ts │ ├── branches │ │ ├── __snapshots__ │ │ │ └── api │ │ │ │ └── branches_list.json │ │ ├── branches.controller.test.ts │ │ ├── branches.controller.ts │ │ └── branches.controller.e2e.ts │ ├── health-check │ │ ├── health-check.controller.ts │ │ └── health-check.controller.test.ts │ ├── tree │ │ ├── tree.controller.e2e.ts │ │ ├── tree.controller.ts │ │ └── __snapshots__ │ │ │ └── tree.controller.e2e.ts.snap │ ├── content │ │ ├── content.controller.e2e.ts │ │ ├── content.controller.ts │ │ └── __snapshots__ │ │ │ └── content.controller.e2e.ts.snap │ ├── commits │ │ ├── commits.controller.e2e.ts │ │ ├── __snapshots__ │ │ │ └── api │ │ │ │ └── commit_list_master.json │ │ ├── commits.controller.test.ts │ │ └── commits.controller.ts │ └── compare │ │ └── compare.controller.ts ├── main.ts ├── app.ts └── app.module.ts ├── test ├── jest-setup.ts └── e2e │ ├── index.ts │ ├── utils.ts │ ├── e2e-server.ts │ └── custom-matchers.ts ├── .prettierrc.yml ├── .dockerignore ├── config ├── tsconfig.build.json ├── jest.e2e.config.js └── jest.dev.config.js ├── .vscode ├── launch.json └── settings.json ├── jest.config.js ├── Dockerfile ├── .gitignore ├── LICENSE ├── .ci ├── ci.yml └── publish.yml ├── tslint.json ├── tsconfig.json ├── package.json ├── README.md └── swagger-spec.json /.gitattributes: -------------------------------------------------------------------------------- 1 | swagger-spec.json eol=lf -------------------------------------------------------------------------------- /sdk/test/typescript/.gitignore: -------------------------------------------------------------------------------- 1 | package-lock.json -------------------------------------------------------------------------------- /src/config/test.ts: -------------------------------------------------------------------------------- 1 | export const testConfig = {}; 2 | -------------------------------------------------------------------------------- /test/jest-setup.ts: -------------------------------------------------------------------------------- 1 | import "reflect-metadata"; 2 | -------------------------------------------------------------------------------- /src/config/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./configuration"; 2 | -------------------------------------------------------------------------------- /src/core/context/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./context"; 2 | -------------------------------------------------------------------------------- /src/core/logger/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./logger"; 2 | -------------------------------------------------------------------------------- /src/models/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./repo-reference"; 2 | -------------------------------------------------------------------------------- /src/services/db/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./db.service"; 2 | -------------------------------------------------------------------------------- /src/services/fs/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./fs.service"; 2 | -------------------------------------------------------------------------------- /.prettierrc.yml: -------------------------------------------------------------------------------- 1 | trailingComma: "all" 2 | printWidth: 120 3 | -------------------------------------------------------------------------------- /src/services/http/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./http.service"; 2 | -------------------------------------------------------------------------------- /src/services/repo/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./repo.service"; 2 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | tmp 2 | bin 3 | buildcache 4 | node_modules 5 | .git -------------------------------------------------------------------------------- /src/core/models/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./models-decorators"; 2 | -------------------------------------------------------------------------------- /src/definitions.d.ts: -------------------------------------------------------------------------------- 1 | type StringMap = { [key: string]: T }; 2 | -------------------------------------------------------------------------------- /src/services/branch/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./branch.service"; 2 | -------------------------------------------------------------------------------- /src/services/commit/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./commit.service"; 2 | -------------------------------------------------------------------------------- /src/services/compare/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./compare.service"; 2 | -------------------------------------------------------------------------------- /src/services/content/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./content.service"; 2 | -------------------------------------------------------------------------------- /src/services/repo/local-repo/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./local-repo"; 2 | -------------------------------------------------------------------------------- /src/services/disk-usage/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./disk-usage.service"; 2 | -------------------------------------------------------------------------------- /src/services/repo-cleanup/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./repo-cleanup.service"; 2 | -------------------------------------------------------------------------------- /src/services/repo-index/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./repo-index.service"; 2 | -------------------------------------------------------------------------------- /src/services/permission/cache/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./permission-cache.service"; 2 | -------------------------------------------------------------------------------- /src/core/telemetry/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./factory"; 2 | export * from "./telemetry"; 3 | -------------------------------------------------------------------------------- /sdk/src/typescript/src/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./models"; 2 | export * from "./gITRestAPI"; 3 | -------------------------------------------------------------------------------- /src/services/repo/mutex/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./mutex"; 2 | export * from "./state-mutex"; 3 | -------------------------------------------------------------------------------- /src/core/repo-auth/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./repo-auth"; 2 | export * from "./repo-auth-decorators"; 3 | -------------------------------------------------------------------------------- /test/e2e/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./e2e-server"; 2 | export * from "./utils"; 3 | import "./custom-matchers"; 4 | -------------------------------------------------------------------------------- /src/middlewares/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./logger-interceptor"; 2 | export * from "./context/context.middleware"; 3 | -------------------------------------------------------------------------------- /src/services/permission/permissions.ts: -------------------------------------------------------------------------------- 1 | export enum GitRemotePermission { 2 | None, 3 | Read, 4 | Write, 5 | } 6 | -------------------------------------------------------------------------------- /src/services/permission/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./permission.service"; 2 | export * from "./cache"; 3 | export * from "./permissions"; 4 | -------------------------------------------------------------------------------- /src/config/development.ts: -------------------------------------------------------------------------------- 1 | import { Configuration } from "./configuration"; 2 | 3 | export const developmentConfig: Partial = {}; 4 | -------------------------------------------------------------------------------- /src/core/pagination/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./pagination"; 2 | export * from "./pagination-decorators"; 3 | export * from "./paginated-response"; 4 | -------------------------------------------------------------------------------- /src/utils/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./secure-utils"; 2 | export * from "./git-utils"; 3 | export * from "./repo-utils"; 4 | export * from "./misc-utils"; 5 | -------------------------------------------------------------------------------- /src/config/production.ts: -------------------------------------------------------------------------------- 1 | import { Configuration } from "./configuration"; 2 | 3 | export const productionConfig: Partial = { 4 | env: "production", 5 | }; 6 | -------------------------------------------------------------------------------- /src/core/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./repo-auth"; 2 | export * from "./logger"; 3 | export * from "./pagination"; 4 | export * from "./telemetry"; 5 | export * from "./models"; 6 | export * from "./context"; 7 | -------------------------------------------------------------------------------- /src/services/app.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from "@nestjs/common"; 2 | 3 | @Injectable() 4 | export class AppService { 5 | public getHello(): string { 6 | return "Hello World!"; 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/utils/repo-utils.ts: -------------------------------------------------------------------------------- 1 | export const RepoUtils = { 2 | getUrl: (remote: string) => { 3 | const suffix = remote.endsWith(".git") ? "" : ".git"; 4 | return `https://${remote}${suffix}`; 5 | }, 6 | }; 7 | -------------------------------------------------------------------------------- /src/core/telemetry/noop-telemetry.ts: -------------------------------------------------------------------------------- 1 | import { Metric, Telemetry } from "./telemetry"; 2 | 3 | export class NoopTelemetry extends Telemetry { 4 | public emitMetric(_: Metric): void { 5 | // Noop 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /sdk/config.yaml: -------------------------------------------------------------------------------- 1 | input-file: ../swagger-spec.json 2 | typescript: 3 | package-name: "git-rest-api" 4 | payload-flattening-threshold: 0 5 | generate-package-json: false 6 | output-folder: "out/typescript" 7 | enum-types: true -------------------------------------------------------------------------------- /src/scripts/generate-swagger-specs.ts: -------------------------------------------------------------------------------- 1 | import { saveSwagger } from "./swagger-generation"; 2 | 3 | void saveSwagger().then(() => { 4 | // tslint:disable-next-line: no-console 5 | console.log("Generated the specs"); 6 | }); 7 | -------------------------------------------------------------------------------- /src/core/telemetry/telemetry.ts: -------------------------------------------------------------------------------- 1 | export interface Metric { 2 | name: string; 3 | value: number; 4 | dimensions?: StringMap; 5 | } 6 | 7 | export abstract class Telemetry { 8 | public abstract emitMetric(metric: Metric): void; 9 | } 10 | -------------------------------------------------------------------------------- /config/tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "rootDir": "../src", 4 | "outDir": "../bin" 5 | }, 6 | "extends": "../tsconfig.json", 7 | "exclude": ["node_modules", "bin", "../test", "../src/**/*.test.ts", "../src/**/*.e2e.ts"] 8 | } 9 | -------------------------------------------------------------------------------- /src/utils/secure-utils.ts: -------------------------------------------------------------------------------- 1 | import crypto from "crypto"; 2 | 3 | export const SecureUtils = { 4 | sha512: (key: string) => { 5 | const hash = crypto.createHash("sha512"); 6 | hash.update(key); 7 | return hash.digest("hex"); 8 | }, 9 | }; 10 | -------------------------------------------------------------------------------- /src/dtos/git-dir-object-content.ts: -------------------------------------------------------------------------------- 1 | import { GitObjectContent } from "./git-object-content"; 2 | 3 | export class GitDirObjectContent extends GitObjectContent { 4 | constructor(gitObjectContent: GitDirObjectContent) { 5 | super(gitObjectContent); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/dtos/git-commit-ref.ts: -------------------------------------------------------------------------------- 1 | import { ApiProperty } from "@nestjs/swagger"; 2 | 3 | export class GitCommitRef { 4 | @ApiProperty({ type: String }) 5 | public sha: string; 6 | 7 | constructor(commit: GitCommitRef) { 8 | this.sha = commit.sha; 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/controllers/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./commits/commits.controller"; 2 | export * from "./health-check/health-check.controller"; 3 | export * from "./branches/branches.controller"; 4 | export * from "./compare/compare.controller"; 5 | export * from "./content/content.controller"; 6 | -------------------------------------------------------------------------------- /src/dtos/git-submodule-object-content.ts: -------------------------------------------------------------------------------- 1 | import { GitObjectContent } from "./git-object-content"; 2 | 3 | export class GitSubmoduleObjectContent extends GitObjectContent { 4 | constructor(gitObjectContent: GitSubmoduleObjectContent) { 5 | super(gitObjectContent); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/services/http/http.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from "@nestjs/common"; 2 | import fetch, { RequestInit, Response } from "node-fetch"; 3 | 4 | @Injectable() 5 | export class HttpService { 6 | public fetch(uri: string, init?: RequestInit): Promise { 7 | return fetch(uri, init); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /config/jest.e2e.config.js: -------------------------------------------------------------------------------- 1 | // @ts-check 2 | 3 | const mainConfig = require("../jest.config"); 4 | 5 | /** @type {jest.InitialOptions} */ 6 | const config = { 7 | ...mainConfig, 8 | testMatch: ["/src/**/*.e2e.ts"], 9 | rootDir: "../", 10 | collectCoverage: false, 11 | }; 12 | 13 | module.exports = config; 14 | -------------------------------------------------------------------------------- /src/controllers/branches/__snapshots__/api/branches_list.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "master", 4 | "commit": { 5 | "sha": "5a999cf552a4bbb5bc21dfabe660e815283f9244" 6 | } 7 | }, 8 | { 9 | "name": "stable", 10 | "commit": { 11 | "sha": "04ccae1ed572f3cdc277c4df2ac73130efbbba3a" 12 | } 13 | } 14 | ] -------------------------------------------------------------------------------- /src/models/repo-reference.ts: -------------------------------------------------------------------------------- 1 | import { Column, Entity, PrimaryColumn } from "typeorm"; 2 | 3 | @Entity({ name: "repo_references" }) 4 | export class RepoReferenceRecord { 5 | @PrimaryColumn() 6 | public path!: string; 7 | 8 | @Column("bigint") 9 | public lastUse!: number; 10 | 11 | @Column("bigint", { nullable: true }) 12 | public lastFetch?: number; 13 | } 14 | -------------------------------------------------------------------------------- /test/e2e/utils.ts: -------------------------------------------------------------------------------- 1 | import path from "path"; 2 | import rimraf from "rimraf"; 3 | import { promisify } from "util"; 4 | 5 | const rm = promisify(rimraf); 6 | 7 | const dataFolder = path.resolve(path.join(__dirname, "../../tmp")); 8 | 9 | export async function deleteLocalRepo(name: string) { 10 | await rm(path.join(dataFolder, "repos", encodeURIComponent(name))); 11 | } 12 | -------------------------------------------------------------------------------- /src/controllers/health-check/health-check.controller.ts: -------------------------------------------------------------------------------- 1 | import { Controller, Get } from "@nestjs/common"; 2 | import { ApiOperation } from "@nestjs/swagger"; 3 | 4 | @Controller("/health") 5 | export class HealthCheckController { 6 | @Get("/alive") 7 | @ApiOperation({ summary: "Check alive", operationId: "health_checkAlive" }) 8 | public async getAlive() { 9 | return "alive"; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/main.ts: -------------------------------------------------------------------------------- 1 | import { createApp } from "./app"; 2 | import { Logger } from "./core"; 3 | 4 | async function bootstrap() { 5 | const { app } = await createApp(); 6 | await app.listen(3009); 7 | } 8 | 9 | bootstrap().catch(error => { 10 | const logger = new Logger("Bootstrap"); 11 | // tslint:disable-next-line: no-console 12 | logger.error("Error in app", error); 13 | process.exit(1); 14 | }); 15 | -------------------------------------------------------------------------------- /sdk/test/typescript/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "git-rest-api-sdk-test", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "ts-node test.ts" 8 | }, 9 | "dependencies": { 10 | "git-rest-api-sdk": "../../out/typescript", 11 | "ts-node": "^8.1.0", 12 | "typescript": "^3.4.5" 13 | }, 14 | "author": "", 15 | "license": "ISC" 16 | } 17 | -------------------------------------------------------------------------------- /src/services/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./app.service"; 2 | export * from "./branch"; 3 | export * from "./commit"; 4 | export * from "./content"; 5 | export * from "./compare"; 6 | export * from "./disk-usage"; 7 | export * from "./fs"; 8 | export * from "./repo"; 9 | export * from "./repo-cleanup"; 10 | export * from "./repo-index"; 11 | export * from "./permission"; 12 | export * from "./http"; 13 | export * from "./db"; 14 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "type": "node", 6 | "request": "launch", 7 | "name": "Run server", 8 | "program": "${workspaceFolder}/src/main.ts", 9 | "outFiles": ["${workspaceFolder}/bin/**/*.js"], 10 | "console": "integratedTerminal", 11 | "sourceMaps": true, 12 | "cwd": "${workspaceFolder}" 13 | } 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /src/dtos/git-branch.ts: -------------------------------------------------------------------------------- 1 | import { ApiProperty } from "@nestjs/swagger"; 2 | 3 | import { GitCommitRef } from "./git-commit-ref"; 4 | 5 | export class GitBranch { 6 | @ApiProperty({ type: String }) 7 | public name: string; 8 | 9 | @ApiProperty({ type: GitCommitRef }) 10 | public commit: GitCommitRef; 11 | 12 | constructor(branch: GitBranch) { 13 | this.name = branch.name; 14 | this.commit = new GitCommitRef(branch.commit); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/scripts/swagger.test.ts: -------------------------------------------------------------------------------- 1 | // import { generateSwagger, getSavedSwagger } from "./swagger-generation"; 2 | 3 | describe("Swagger specs", () => { 4 | it("should be up to date", async () => { 5 | // const [specs, current] = await Promise.all([generateSwagger(), getSavedSwagger()]); 6 | // Swagger specs saved in the repo should match the ones being generated run `npm run swagger:gen` to regenerate 7 | // expect(specs).toEqual(current); 8 | }); 9 | }); 10 | -------------------------------------------------------------------------------- /src/dtos/git-file-object-without-content.ts: -------------------------------------------------------------------------------- 1 | import { ApiProperty } from "@nestjs/swagger"; 2 | 3 | import { GitObjectContent } from "./git-object-content"; 4 | 5 | export class GitFileObjectWithoutContent extends GitObjectContent { 6 | @ApiProperty({ type: String }) 7 | public encoding: string; 8 | 9 | constructor(gitObjectContent: GitFileObjectWithoutContent) { 10 | super(gitObjectContent); 11 | this.encoding = gitObjectContent.encoding; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/dtos/git-signature.ts: -------------------------------------------------------------------------------- 1 | import { ApiProperty } from "@nestjs/swagger"; 2 | 3 | export class GitSignature { 4 | @ApiProperty({ type: String }) 5 | public name: string; 6 | @ApiProperty({ type: String }) 7 | public email: string; 8 | @ApiProperty({ type: String, format: "date-time" }) 9 | public date: Date; 10 | 11 | constructor(sig: GitSignature) { 12 | this.name = sig.name; 13 | this.email = sig.email; 14 | this.date = sig.date; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/controllers/health-check/health-check.controller.test.ts: -------------------------------------------------------------------------------- 1 | import { HealthCheckController } from "./health-check.controller"; 2 | 3 | describe("HealthCheckController", () => { 4 | let controller: HealthCheckController; 5 | 6 | beforeEach(() => { 7 | controller = new HealthCheckController(); 8 | }); 9 | 10 | describe("alive", () => { 11 | it('should return alive"', async () => { 12 | expect(await controller.getAlive()).toBe("alive"); 13 | }); 14 | }); 15 | }); 16 | -------------------------------------------------------------------------------- /src/dtos/git-file-object-with-content.ts: -------------------------------------------------------------------------------- 1 | import { ApiProperty } from "@nestjs/swagger"; 2 | 3 | import { GitFileObjectWithoutContent } from "./git-file-object-without-content"; 4 | 5 | export class GitFileObjectWithContent extends GitFileObjectWithoutContent { 6 | @ApiProperty({ type: String }) 7 | public content: string; 8 | 9 | constructor(gitObjectContent: GitFileObjectWithContent) { 10 | super(gitObjectContent); 11 | this.content = gitObjectContent.content; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/controllers/tree/tree.controller.e2e.ts: -------------------------------------------------------------------------------- 1 | import { TEST_REPO, e2eClient } from "../../../test/e2e"; 2 | 3 | describe("Test content controller", () => { 4 | const base = `/repos/${TEST_REPO}/tree`; 5 | test.each(["", "/", "/README.md", "/dir1"])(`for path '${base}%s'`, async tail => { 6 | const response = await e2eClient.fetch(`${base}${tail}`); 7 | expect(response.status).toEqual(200); 8 | 9 | const body = await response.json(); 10 | expect(body).toMatchSnapshot(); 11 | }); 12 | }); 13 | -------------------------------------------------------------------------------- /src/dtos/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./git-branch"; 2 | export * from "./git-commit-ref"; 3 | export * from "./git-commit"; 4 | export * from "./git-contents"; 5 | export * from "./git-diff"; 6 | export * from "./git-dir-object-content"; 7 | export * from "./git-file-diff"; 8 | export * from "./git-file-object-with-content"; 9 | export * from "./git-file-object-without-content"; 10 | export * from "./git-object-content"; 11 | export * from "./git-signature"; 12 | export * from "./git-submodule-object-content"; 13 | export * from "./git-tree"; 14 | -------------------------------------------------------------------------------- /config/jest.dev.config.js: -------------------------------------------------------------------------------- 1 | // @ts-check 2 | 3 | const mainConfig = require("../jest.config"); 4 | 5 | /** @type {jest.InitialOptions} */ 6 | const config = { 7 | ...mainConfig, 8 | globals: { 9 | "ts-jest": { 10 | tsConfig: "tsconfig.json", 11 | diagnostics: { warnOnly: true } // Make typescript show errors as warning while writing test. This is specially for noUnusedLocals which is always preventing test to run. 12 | } 13 | }, 14 | rootDir: "../", 15 | verbose: false 16 | }; 17 | 18 | module.exports = config; 19 | -------------------------------------------------------------------------------- /sdk/src/typescript/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "es6", 4 | "moduleResolution": "node", 5 | "strict": true, 6 | "target": "es5", 7 | "sourceMap": true, 8 | "declarationMap": true, 9 | "esModuleInterop": true, 10 | "allowSyntheticDefaultImports": true, 11 | "forceConsistentCasingInFileNames": true, 12 | "lib": ["es6"], 13 | "declaration": true, 14 | "outDir": "./esm", 15 | "importHelpers": true 16 | }, 17 | "include": ["./src/**/*.ts"], 18 | "exclude": ["node_modules"] 19 | } 20 | -------------------------------------------------------------------------------- /test/e2e/e2e-server.ts: -------------------------------------------------------------------------------- 1 | import fetch, { RequestInit, Response } from "node-fetch"; 2 | 3 | const testUrl = process.env.GIT_REST_API_E2E_ENDPOINT || "http://localhost:3009"; 4 | jest.setTimeout(60_000); 5 | 6 | class E2EClient { 7 | public fetch(uri: string, init?: RequestInit): Promise { 8 | return fetch(`${testUrl}${uri}`, init); 9 | } 10 | } 11 | 12 | export const e2eClient = new E2EClient(); 13 | export const UNENCODED_TEST_REPO = "github.com/test-repo-billy/git-api-tests"; 14 | export const TEST_REPO = encodeURIComponent(UNENCODED_TEST_REPO); 15 | -------------------------------------------------------------------------------- /src/controllers/content/content.controller.e2e.ts: -------------------------------------------------------------------------------- 1 | import { TEST_REPO, e2eClient } from "../../../test/e2e"; 2 | 3 | describe("Test content controller", () => { 4 | const base = `/repos/${TEST_REPO}/contents`; 5 | test.each(["", "/", "/README.md", "/dir1", "/dir1?recursive=true", "?recursive=true"])( 6 | `for path '${base}%s'`, 7 | async tail => { 8 | const response = await e2eClient.fetch(`${base}${tail}`); 9 | expect(response.status).toEqual(200); 10 | 11 | const body = await response.json(); 12 | expect(body).toMatchSnapshot(); 13 | }, 14 | ); 15 | }); 16 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "typescript.tsdk": "node_modules/typescript/lib", 3 | "search.exclude": { 4 | "**/node_modules": true, 5 | "bin": true, 6 | "buildcache": true, 7 | "coverage": true 8 | }, 9 | "files.exclude": { 10 | "**/.git": true, 11 | "**/.svn": true, 12 | "**/.hg": true, 13 | "**/CVS": true, 14 | "**/.DS_Store": true, 15 | "sdk/out/**/node_modules": true 16 | }, 17 | "editor.defaultFormatter": "esbenp.prettier-vscode", 18 | "typescript.disableAutomaticTypeAcquisition": true // Cause issue with locking folder on windows. Types should be installed anyway in package.json 19 | } 20 | -------------------------------------------------------------------------------- /src/utils/git-utils.ts: -------------------------------------------------------------------------------- 1 | export interface RemoteRef { 2 | ref: string; 3 | remote: string; 4 | } 5 | 6 | export const GitUtils = { 7 | /** 8 | * Parse a remote reference 9 | * Can be in the follow format: 10 | * - "{ref}" 11 | * - "{remote}:{ref}" 12 | * @param name Reference name 13 | */ 14 | parseRemoteReference: (name: string, defaultRemote: string): RemoteRef => { 15 | if (name.includes(":")) { 16 | const [remote, ref] = name.split(":"); 17 | return { 18 | remote, 19 | ref, 20 | }; 21 | } else { 22 | return { ref: name, remote: defaultRemote }; 23 | } 24 | }, 25 | }; 26 | -------------------------------------------------------------------------------- /src/core/pagination/pagination.ts: -------------------------------------------------------------------------------- 1 | export interface Pagination { 2 | page?: number; 3 | } 4 | 5 | /** 6 | * Return the given page for the given pagination 7 | * @param pagination 8 | */ 9 | export function getPage(pagination: Pagination | undefined) { 10 | return pagination === undefined || pagination.page === undefined ? 1 : Math.max(pagination.page, 1); 11 | } 12 | 13 | /** 14 | * Return the number of items to skip using the given pagination object 15 | * @param pagination 16 | */ 17 | export function getPaginationSkip(pagination: Pagination | undefined, perPage: number) { 18 | const page = getPage(pagination); 19 | return (page - 1) * perPage; 20 | } 21 | -------------------------------------------------------------------------------- /src/core/context/context.ts: -------------------------------------------------------------------------------- 1 | import cls from "cls-hooked"; 2 | const ns = cls.createNamespace("Open API Hub Context"); 3 | 4 | export interface HubContext { 5 | requestId: string; 6 | } 7 | 8 | export function getContext(key: K): HubContext[K] | undefined { 9 | if (!ns || !ns.active) { 10 | return undefined; 11 | } 12 | 13 | return ns.get(key); 14 | } 15 | 16 | export function setContext(key: K, value: HubContext[K]) { 17 | if (!ns || !ns.active) { 18 | return; 19 | } 20 | 21 | return ns.set(key, value); 22 | } 23 | 24 | export function runInContext(func: (...args: unknown[]) => unknown) { 25 | ns.run(func); 26 | } 27 | -------------------------------------------------------------------------------- /src/services/db/db.service.ts: -------------------------------------------------------------------------------- 1 | import { createConnection } from "typeorm"; 2 | 3 | import { Configuration } from "../../config"; 4 | import { RepoReferenceRecord } from "../../models"; 5 | 6 | /** 7 | * Create a DB Connection to sqlite and sync the schema. 8 | * Can be used as NestJS async factory and then have the Connection as a injectable for other service. 9 | */ 10 | export async function createDBConnection(configuration: Configuration) { 11 | const connection = await createConnection({ 12 | type: "sqlite", 13 | database: `${configuration.dataDir}/data.sqlite`, 14 | entities: [RepoReferenceRecord], 15 | }); 16 | await connection.synchronize(); 17 | return connection; 18 | } 19 | -------------------------------------------------------------------------------- /sdk/test/typescript/test.ts: -------------------------------------------------------------------------------- 1 | import { GITRestAPI, GitBranch } from "git-rest-api-sdk"; 2 | // tslint:disable: no-console 3 | 4 | const sdk = new GITRestAPI({ baseUri: "http://localhost:3009" }); 5 | 6 | async function run() { 7 | const branches: GitBranch[] = await sdk.branches.list("github.com/Azure/BatchExplorer"); 8 | console.log("Branches:"); 9 | 10 | for (const branch of branches) { 11 | console.log(` - ${branch.name} ${branch.commit.sha}`); 12 | } 13 | 14 | const response = await sdk.commits.list("github.com/Azure/BatchExplorer"); 15 | console.log("Total commits", response.xTotalCount); 16 | } 17 | 18 | run().catch(e => { 19 | console.error(e); 20 | process.exit(1); 21 | }); 22 | -------------------------------------------------------------------------------- /src/services/fs/fs.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from "@nestjs/common"; 2 | import fs from "fs"; 3 | import rimraf from "rimraf"; 4 | import { promisify } from "util"; 5 | 6 | const rm = promisify(rimraf); 7 | 8 | @Injectable() 9 | export class FSService { 10 | public async exists(path: string): Promise { 11 | try { 12 | await fs.promises.access(path); 13 | return true; 14 | } catch { 15 | return false; 16 | } 17 | } 18 | 19 | public async mkdir(path: string): Promise { 20 | await fs.promises.mkdir(path, { recursive: true }); 21 | return path; 22 | } 23 | 24 | public async rm(path: string): Promise { 25 | await rm(path); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/core/models/models-decorators.ts: -------------------------------------------------------------------------------- 1 | import { ApiProperty } from "@nestjs/swagger"; 2 | import { SwaggerEnumType } from "@nestjs/swagger/dist/types/swagger-enum.type"; 3 | 4 | /** 5 | * Decorator to let swagger know about all the pagination properties available 6 | */ 7 | export function ApiModelEnum(metadata: { [enumName: string]: SwaggerEnumType }): PropertyDecorator { 8 | const enumName = Object.keys(metadata)[0]; 9 | if (!enumName) { 10 | throw new Error("You must provide an enum to ApiModelEnum. `@ApiModelEnum({ Myenum })`"); 11 | } 12 | return ApiProperty({ 13 | enum: metadata[enumName], 14 | // Adding custom properties for auto rest 15 | "x-ms-enum": { name: enumName }, 16 | } as any); 17 | } 18 | -------------------------------------------------------------------------------- /src/dtos/git-object-content.ts: -------------------------------------------------------------------------------- 1 | import { ApiProperty } from "@nestjs/swagger"; 2 | 3 | export class GitObjectContent { 4 | @ApiProperty({ type: String }) 5 | public type: string; 6 | @ApiProperty({ type: Number }) 7 | public size: number; 8 | @ApiProperty({ type: String }) 9 | public name: string; 10 | @ApiProperty({ type: String }) 11 | public path: string; 12 | @ApiProperty({ type: String }) 13 | public sha: string; 14 | 15 | constructor(gitObjectContent: GitObjectContent) { 16 | this.type = gitObjectContent.type; 17 | this.size = gitObjectContent.size; 18 | this.name = gitObjectContent.name; 19 | this.path = gitObjectContent.path; 20 | this.sha = gitObjectContent.sha; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/controllers/commits/commits.controller.e2e.ts: -------------------------------------------------------------------------------- 1 | import { TEST_REPO, e2eClient } from "../../../test/e2e"; 2 | 3 | describe("Test commits controller", () => { 4 | it("List commits in the test repo", async () => { 5 | const response = await e2eClient.fetch(`/repos/${TEST_REPO}/commits`); 6 | expect(response.status).toEqual(200); 7 | const content = await response.json(); 8 | expect(content).toMatchPayload("commit_list_master"); 9 | }); 10 | 11 | it("returns empty array if asking for page that doesn't exists", async () => { 12 | const response = await e2eClient.fetch(`/repos/${TEST_REPO}/commits?page=999999`); 13 | expect(response.status).toEqual(200); 14 | const content = await response.json(); 15 | expect(content).toEqual([]); 16 | }); 17 | }); 18 | -------------------------------------------------------------------------------- /src/core/telemetry/factory.ts: -------------------------------------------------------------------------------- 1 | import { Configuration } from "../../config"; 2 | import { Logger } from "../logger"; 3 | import { NoopTelemetry } from "./noop-telemetry"; 4 | import { StatsdTelemetry } from "./statsd-telemetry"; 5 | import { Telemetry } from "./telemetry"; 6 | 7 | export function createTelemetry(config: Configuration): Telemetry { 8 | const logger = new Logger("TelemetryFactory"); 9 | const instance = getTelmetryInstance(config); 10 | logger.info(`Resolving telemetry engine from configuration: ${instance.constructor.name}`); 11 | return instance; 12 | } 13 | 14 | function getTelmetryInstance(config: Configuration) { 15 | if (config.statsd.host && config.statsd.port) { 16 | return new StatsdTelemetry(config); 17 | } else { 18 | return new NoopTelemetry(); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/core/telemetry/statsd-telemetry.ts: -------------------------------------------------------------------------------- 1 | import { StatsD } from "hot-shots"; 2 | 3 | import { Configuration } from "../../config"; 4 | import { Metric, Telemetry } from "./telemetry"; 5 | 6 | export class StatsdTelemetry extends Telemetry { 7 | private instance: StatsD; 8 | 9 | constructor(private config: Configuration) { 10 | super(); 11 | const { host, port } = this.config.statsd; 12 | this.instance = new StatsD({ host, port }); 13 | } 14 | 15 | public emitMetric(metric: Metric): void { 16 | const stat = JSON.stringify({ 17 | Metric: metric.name, 18 | Namespace: this.config.serviceName, 19 | Dims: { 20 | ...metric.dimensions, 21 | env: this.config.env, 22 | }, 23 | }); 24 | 25 | this.instance.gauge(stat, metric.value); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /sdk/src/typescript/rollup.config.js: -------------------------------------------------------------------------------- 1 | import nodeResolve from "rollup-plugin-node-resolve"; 2 | /** 3 | * @type {import('rollup').RollupFileOptions} 4 | */ 5 | const config = { 6 | input: './esm/index.js', 7 | external: ["@azure/ms-rest-js", "@azure/ms-rest-azure-js"], 8 | output: { 9 | file: "./dist/bundle.js", 10 | format: "umd", 11 | name: "Bundle", 12 | sourcemap: true, 13 | globals: { 14 | "@azure/ms-rest-js": "msRest", 15 | "@azure/ms-rest-azure-js": "msRestAzure" 16 | }, 17 | banner: `/* 18 | * Code generated by Microsoft (R) AutoRest Code Generator. 19 | * Changes may cause incorrect behavior and will be lost if the code is 20 | * regenerated. 21 | */` 22 | }, 23 | plugins: [ 24 | nodeResolve({ module: true }) 25 | ] 26 | }; 27 | export default config; 28 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | // @ts-check 2 | 3 | /** @type {jest.InitialOptions} */ 4 | const config = { 5 | transform: { 6 | "^.+\\.ts$": "ts-jest", 7 | }, 8 | moduleFileExtensions: ["ts", "js", "json", "node"], 9 | moduleNameMapper: {}, 10 | collectCoverage: true, 11 | collectCoverageFrom: ["src/**/*.ts", "!**/node_modules/**"], 12 | coverageReporters: ["json", "lcov", "cobertura", "text", "html", "clover"], 13 | coveragePathIgnorePatterns: ["/node_modules/", ".*/test/.*"], 14 | modulePathIgnorePatterns: ['/sdk'], 15 | globals: { 16 | "ts-jest": { 17 | tsConfig: "tsconfig.json", 18 | }, 19 | }, 20 | setupFiles: ["/test/jest-setup.ts"], 21 | testMatch: ["/src/**/*.test.ts"], 22 | verbose: true, 23 | testEnvironment: "node", 24 | }; 25 | 26 | module.exports = config; 27 | -------------------------------------------------------------------------------- /src/dtos/git-commit.ts: -------------------------------------------------------------------------------- 1 | import { ApiProperty } from "@nestjs/swagger"; 2 | 3 | import { GitCommitRef } from "./git-commit-ref"; 4 | import { GitSignature } from "./git-signature"; 5 | 6 | export class GitCommit extends GitCommitRef { 7 | @ApiProperty({ type: String }) 8 | public message: string; 9 | 10 | @ApiProperty({ type: GitSignature }) 11 | public author: GitSignature; 12 | 13 | @ApiProperty({ type: GitSignature }) 14 | public committer: GitSignature; 15 | 16 | @ApiProperty({ type: GitCommitRef, isArray: true }) 17 | public parents: GitCommitRef[]; 18 | 19 | constructor(commit: GitCommit) { 20 | super(commit); 21 | this.message = commit.message; 22 | this.author = commit.author; 23 | this.committer = commit.committer; 24 | this.parents = commit.parents.map(x => new GitCommitRef(x)); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/utils/misc-utils.ts: -------------------------------------------------------------------------------- 1 | export const notUndefined = (x: T | undefined): x is T => x !== undefined; 2 | export const delay = (timeout?: number) => new Promise(r => setTimeout(r, timeout)); 3 | export const parseBooleanFromURLParam = (bool: string | undefined) => bool === "" || bool === "true"; 4 | 5 | export class Deferred { 6 | public promise: Promise; 7 | public hasCompleted = false; 8 | public resolve!: T extends void ? () => void : (v: T) => void; 9 | public reject!: (e: unknown) => void; 10 | 11 | constructor() { 12 | this.promise = new Promise((resolve, reject) => { 13 | this.reject = x => { 14 | this.hasCompleted = true; 15 | reject(x); 16 | }; 17 | this.resolve = ((x: T) => { 18 | this.hasCompleted = true; 19 | resolve(x); 20 | }) as any; 21 | }); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/app.ts: -------------------------------------------------------------------------------- 1 | import { NestFactory } from "@nestjs/core"; 2 | import { DocumentBuilder, SwaggerModule } from "@nestjs/swagger"; 3 | import helmet from "helmet"; 4 | 5 | import { AppModule } from "./app.module"; 6 | import { NestLogger } from "./core/logger/nest-logger"; 7 | 8 | export async function createApp() { 9 | const app = await NestFactory.create(AppModule, { 10 | logger: new NestLogger(), 11 | }); 12 | app.enableCors(); 13 | app.use(helmet()); 14 | 15 | const options = new DocumentBuilder() 16 | .setTitle("GIT Rest API") 17 | .setDescription("Rest api to run operation on git repositories") 18 | .setVersion("1.0") 19 | .addServer("http://") 20 | .addServer("https://") 21 | .build(); 22 | const document = SwaggerModule.createDocument(app, options); 23 | SwaggerModule.setup("swagger", app, document); 24 | return { app, document }; 25 | } 26 | -------------------------------------------------------------------------------- /src/dtos/git-contents.ts: -------------------------------------------------------------------------------- 1 | import { ApiProperty } from "@nestjs/swagger"; 2 | 3 | import { GitDirObjectContent } from "./git-dir-object-content"; 4 | import { GitFileObjectWithContent } from "./git-file-object-with-content"; 5 | import { GitSubmoduleObjectContent } from "./git-submodule-object-content"; 6 | 7 | export class GitTree { 8 | @ApiProperty({ type: GitDirObjectContent, isArray: true }) 9 | public dirs: GitDirObjectContent[]; 10 | @ApiProperty({ type: GitFileObjectWithContent, isArray: true }) 11 | public files: GitFileObjectWithContent[]; 12 | @ApiProperty({ type: GitSubmoduleObjectContent, isArray: true }) 13 | public submodules: GitSubmoduleObjectContent[]; 14 | 15 | constructor(gitObjectContent: GitTree) { 16 | this.dirs = gitObjectContent.dirs; 17 | this.files = gitObjectContent.files; 18 | this.submodules = gitObjectContent.submodules; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/dtos/git-tree.ts: -------------------------------------------------------------------------------- 1 | import { ApiProperty } from "@nestjs/swagger"; 2 | 3 | import { GitDirObjectContent } from "./git-dir-object-content"; 4 | import { GitFileObjectWithoutContent } from "./git-file-object-without-content"; 5 | import { GitSubmoduleObjectContent } from "./git-submodule-object-content"; 6 | 7 | export class GitContents { 8 | @ApiProperty({ type: GitDirObjectContent, isArray: true }) 9 | public dirs: GitDirObjectContent[]; 10 | @ApiProperty({ type: GitFileObjectWithoutContent, isArray: true }) 11 | public files: GitFileObjectWithoutContent[]; 12 | @ApiProperty({ type: GitSubmoduleObjectContent, isArray: true }) 13 | public submodules: GitSubmoduleObjectContent[]; 14 | 15 | constructor(gitObjectContent: GitContents) { 16 | this.dirs = gitObjectContent.dirs; 17 | this.files = gitObjectContent.files; 18 | this.submodules = gitObjectContent.submodules; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/controllers/branches/branches.controller.test.ts: -------------------------------------------------------------------------------- 1 | import { RepoAuth } from "../../core"; 2 | import { BranchesController } from "./branches.controller"; 3 | 4 | const b1 = { 5 | name: "master", 6 | commit: { 7 | sha: "sha1", 8 | }, 9 | }; 10 | 11 | const b2 = { 12 | name: "master", 13 | commit: { 14 | sha: "sha2", 15 | }, 16 | }; 17 | 18 | describe("BranchesController", () => { 19 | let controller: BranchesController; 20 | const branchServiceSpy = { 21 | list: jest.fn(() => [b1, b2]), 22 | }; 23 | 24 | beforeEach(() => { 25 | jest.clearAllMocks(); 26 | controller = new BranchesController(branchServiceSpy as any); 27 | }); 28 | 29 | it("list the branches", async () => { 30 | const auth = new RepoAuth(); 31 | const branches = await controller.list("github.com/Azure/git-rest-api", auth); 32 | 33 | expect(branches).toEqual([b1, b2]); 34 | }); 35 | }); 36 | -------------------------------------------------------------------------------- /src/middlewares/context/context.middleware.ts: -------------------------------------------------------------------------------- 1 | import { Injectable, NestMiddleware } from "@nestjs/common"; 2 | import { NextFunction, Request, Response } from "express"; 3 | import uuid from "uuid/v4"; 4 | 5 | import { runInContext, setContext } from "../../core"; 6 | 7 | @Injectable() 8 | export class ContextMiddleware implements NestMiddleware { 9 | public use(request: Request, response: Response, next: NextFunction) { 10 | runInContext(() => { 11 | injectRequestId(request, response); 12 | next(); 13 | }); 14 | } 15 | } 16 | 17 | declare global { 18 | namespace Express { 19 | export interface Request { 20 | requestId: string; 21 | } 22 | } 23 | } 24 | 25 | function injectRequestId(request: Request, response: Response) { 26 | const id = uuid(); 27 | request.requestId = id; 28 | 29 | setContext("requestId", id); 30 | response.setHeader("x-request-id", id); 31 | } 32 | -------------------------------------------------------------------------------- /src/controllers/branches/branches.controller.ts: -------------------------------------------------------------------------------- 1 | import { Controller, Get, Param } from "@nestjs/common"; 2 | import { ApiNotFoundResponse, ApiOkResponse, ApiOperation } from "@nestjs/swagger"; 3 | 4 | import { ApiHasPassThruAuth, Auth, RepoAuth } from "../../core"; 5 | import { GitBranch } from "../../dtos"; 6 | import { BranchService } from "../../services"; 7 | 8 | @Controller("/repos/:remote/branches") 9 | export class BranchesController { 10 | constructor(private branchService: BranchService) {} 11 | 12 | @Get() 13 | @ApiHasPassThruAuth() 14 | @ApiOkResponse({ type: GitBranch, isArray: true }) 15 | @ApiNotFoundResponse({}) 16 | @ApiOperation({ summary: "List branches", operationId: "Branches_List" }) 17 | public async list(@Param("remote") remote: string, @Auth() auth: RepoAuth): Promise { 18 | const branches = await this.branchService.list(remote, { auth }); 19 | return branches; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/dtos/git-diff.ts: -------------------------------------------------------------------------------- 1 | import { ApiProperty } from "@nestjs/swagger"; 2 | 3 | import { GitCommit } from "./git-commit"; 4 | import { GitFileDiff } from "./git-file-diff"; 5 | 6 | export class GitDiff { 7 | @ApiProperty() 8 | public headCommit: GitCommit; 9 | @ApiProperty() 10 | public baseCommit: GitCommit; 11 | @ApiProperty() 12 | public mergeBaseCommit: GitCommit; 13 | @ApiProperty() 14 | public totalCommits: number; 15 | @ApiProperty({ type: GitCommit, isArray: true }) 16 | public commits: GitCommit[]; 17 | @ApiProperty({ type: GitFileDiff, isArray: true }) 18 | public files: GitFileDiff[]; 19 | 20 | constructor(obj: GitDiff) { 21 | this.headCommit = obj.headCommit; 22 | this.baseCommit = obj.baseCommit; 23 | this.mergeBaseCommit = obj.mergeBaseCommit; 24 | this.totalCommits = obj.totalCommits; 25 | this.commits = obj.commits; 26 | this.files = obj.files.map(x => new GitFileDiff(x)); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Base Stage 2 | FROM mcr.microsoft.com/mirror/docker/library/ubuntu:18.04 as base 3 | 4 | RUN mkdir /app 5 | WORKDIR /app 6 | 7 | ENV PKG_ADD="curl" 8 | ENV PKG_ADD_BUILD="libkrb5-dev krb5-config gcc g++ make" 9 | RUN apt update && apt upgrade -y && apt install $PKG_ADD -y 10 | RUN curl -sL https://deb.nodesource.com/setup_12.x | bash - && apt install nodejs -y 11 | 12 | # Build Stage 13 | FROM base AS builder 14 | RUN apt install $PKG_ADD_BUILD -y 15 | 16 | COPY package.json package-lock.json .prettierrc.yml ./ 17 | RUN npm ci 18 | 19 | COPY . . 20 | RUN npm run build 21 | ARG SKIP_TEST 22 | RUN if [ -z "${SKIP_TEST}" ]; then npm run test; fi 23 | RUN npm prune --production 24 | 25 | 26 | # Prod Stage 27 | FROM base as prod 28 | 29 | COPY package.json package-lock.json ./ 30 | COPY --from=builder /app/bin ./bin 31 | COPY --from=builder /app/node_modules ./node_modules 32 | 33 | EXPOSE 3009 34 | 35 | ENV NODE_ENV=production 36 | CMD npm run start:prod -------------------------------------------------------------------------------- /src/services/repo/mutex/state-mutex.ts: -------------------------------------------------------------------------------- 1 | import { Lock, LockOptions, Mutex } from "./mutex"; 2 | 3 | /** 4 | * Mutex that also keeps a state depending on the lock 5 | */ 6 | export class StateMutex { 7 | public state: State; 8 | 9 | private mutex = new Mutex(); 10 | 11 | constructor(private readonly idleState: State, initialState: State) { 12 | this.state = initialState; 13 | } 14 | 15 | public lock(state: SharedState, options?: { exclusive: false }): Promise; 16 | public lock(state: T, options: { exclusive: true }): Promise; 17 | public async lock(state: State, options?: LockOptions): Promise { 18 | const lock = await this.mutex.lock(options); 19 | this.state = state; 20 | return { 21 | id: lock.id, 22 | release: () => { 23 | lock.release(); 24 | if (!this.mutex.pending) { 25 | this.state = this.idleState; 26 | } 27 | }, 28 | }; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/controllers/commits/__snapshots__/api/commit_list_master.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "sha": "5a999cf552a4bbb5bc21dfabe660e815283f9244", 4 | "message": "Add directories (#1)\n\n", 5 | "author": { 6 | "name": "billytrend", 7 | "email": "witrend@microsoft.com", 8 | "date": "2019-07-12T22:02:01.000Z" 9 | }, 10 | "committer": { 11 | "name": "Timothee Guerin", 12 | "email": "timothee.guerin@outlook.com", 13 | "date": "2019-07-12T22:02:01.000Z" 14 | }, 15 | "parents": [ 16 | { 17 | "sha": "04ccae1ed572f3cdc277c4df2ac73130efbbba3a" 18 | } 19 | ] 20 | }, 21 | { 22 | "sha": "04ccae1ed572f3cdc277c4df2ac73130efbbba3a", 23 | "message": "Initial commit", 24 | "author": { 25 | "name": "Timothee Guerin", 26 | "email": "timothee.guerin@outlook.com", 27 | "date": "2019-05-23T23:44:59.000Z" 28 | }, 29 | "committer": { 30 | "name": "GitHub", 31 | "email": "noreply@github.com", 32 | "date": "2019-05-23T23:44:59.000Z" 33 | }, 34 | "parents": [] 35 | } 36 | ] -------------------------------------------------------------------------------- /src/dtos/git-file-diff.ts: -------------------------------------------------------------------------------- 1 | import { ApiProperty, ApiPropertyOptional } from "@nestjs/swagger"; 2 | 3 | import { ApiModelEnum } from "../core"; 4 | 5 | export enum PatchStatus { 6 | Unmodified = "unmodified", 7 | Modified = "modified", 8 | Added = "added", 9 | Deleted = "deleted", 10 | Renamed = "renamed", 11 | } 12 | 13 | export class GitFileDiff { 14 | @ApiProperty() 15 | public filename: string; 16 | @ApiProperty() 17 | public sha: string; 18 | @ApiModelEnum({ PatchStatus }) 19 | public status: PatchStatus; 20 | @ApiProperty() 21 | public additions: number; 22 | @ApiProperty() 23 | public deletions: number; 24 | @ApiProperty() 25 | public changes: number; 26 | @ApiPropertyOptional() 27 | public previousFilename?: string; 28 | 29 | constructor(obj: GitFileDiff) { 30 | this.filename = obj.filename; 31 | this.sha = obj.sha; 32 | this.status = obj.status; 33 | this.additions = obj.additions; 34 | this.deletions = obj.deletions; 35 | this.changes = obj.changes; 36 | this.previousFilename = obj.previousFilename; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/controllers/compare/compare.controller.ts: -------------------------------------------------------------------------------- 1 | import { Controller, Get, HttpException, Param } from "@nestjs/common"; 2 | import { ApiNotFoundResponse, ApiOkResponse, ApiOperation } from "@nestjs/swagger"; 3 | 4 | import { ApiHasPassThruAuth, Auth, RepoAuth } from "../../core"; 5 | import { GitDiff } from "../../dtos"; 6 | import { CompareService } from "../../services"; 7 | 8 | @Controller("/repos/:remote/compare") 9 | export class CompareController { 10 | constructor(private compareService: CompareService) {} 11 | 12 | @Get(":base([^/]*)...:head([^/]*)") 13 | @ApiHasPassThruAuth() 14 | @ApiOkResponse({ type: GitDiff }) 15 | @ApiNotFoundResponse({}) 16 | @ApiOperation({ summary: "Compare two commits", operationId: "commits_compare" }) 17 | public async compare( 18 | @Param("remote") remote: string, 19 | @Param("base") base: string, 20 | @Param("head") head: string, 21 | @Auth() auth: RepoAuth, 22 | ) { 23 | const diff = await this.compareService.compare(remote, base, head, { auth }); 24 | if (diff instanceof HttpException) { 25 | throw diff; 26 | } 27 | return diff; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/core/logger/nest-logger.ts: -------------------------------------------------------------------------------- 1 | import { LoggerService } from "@nestjs/common"; 2 | import winston from "winston"; 3 | 4 | import { WINSTON_LOGGER } from "./winston-logger"; 5 | 6 | /** 7 | * Class to handle logs from nest. 8 | * This shouldn't be used directly this is just to route nest logs to winston 9 | */ 10 | export class NestLogger implements LoggerService { 11 | private logger: winston.Logger; 12 | 13 | constructor() { 14 | this.logger = WINSTON_LOGGER; 15 | } 16 | 17 | public log(message: string, context?: string) { 18 | this.logger.info(message, { context }); 19 | } 20 | 21 | public error(message: string, trace?: string, context?: string) { 22 | this.logger.error(message, { 23 | trace, 24 | context, 25 | }); 26 | } 27 | 28 | public warn(message: string, context?: string) { 29 | this.logger.warn(message, { context }); 30 | } 31 | 32 | public debug(message: string, context?: string) { 33 | this.logger.debug(message, { context }); 34 | } 35 | 36 | public verbose(message: string, context?: string) { 37 | this.logger.verbose(message, { context }); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (https://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # Optional npm cache directory 40 | .npm 41 | 42 | # Optional eslint cache 43 | .eslintcache 44 | 45 | # Optional REPL history 46 | .node_repl_history 47 | 48 | # Output of 'npm pack' 49 | *.tgz 50 | 51 | # Yarn Integrity file 52 | .yarn-integrity 53 | 54 | # dotenv environment variables file 55 | .env 56 | 57 | # next.js build output 58 | .next 59 | 60 | bin/ 61 | buildcache/ 62 | tmp/ 63 | # Junit 64 | test-results* 65 | 66 | # Generated sdk folder 67 | sdk/out/ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. All rights reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /src/controllers/commits/commits.controller.test.ts: -------------------------------------------------------------------------------- 1 | import { NotFoundException } from "@nestjs/common"; 2 | 3 | import { RepoAuth } from "../../core"; 4 | import { CommitsController } from "./commits.controller"; 5 | 6 | const c1 = { 7 | sha: "sha1", 8 | }; 9 | 10 | describe("CommitsController", () => { 11 | let controller: CommitsController; 12 | const commitServiceSpy = { 13 | get: jest.fn((_, sha) => (sha === "sha1" ? c1 : undefined)), 14 | }; 15 | 16 | beforeEach(() => { 17 | jest.clearAllMocks(); 18 | controller = new CommitsController(commitServiceSpy as any); 19 | }); 20 | 21 | it("get a commit", async () => { 22 | const auth = new RepoAuth(); 23 | const commit = await controller.get("github.com/Azure/git-rest-api", "sha1", auth); 24 | expect(commitServiceSpy.get).toHaveBeenCalledTimes(1); 25 | expect(commitServiceSpy.get).toHaveBeenCalledWith("github.com/Azure/git-rest-api", "sha1", { auth }); 26 | expect(commit).toEqual(c1); 27 | }); 28 | 29 | it("throw a NotFoundException if commit doesn't exists", async () => { 30 | const auth = new RepoAuth(); 31 | await expect(controller.get("github.com/Azure/git-rest-api", "sha-not-found", auth)).rejects.toThrow( 32 | NotFoundException, 33 | ); 34 | }); 35 | }); 36 | -------------------------------------------------------------------------------- /src/controllers/tree/tree.controller.ts: -------------------------------------------------------------------------------- 1 | import { Controller, Get, HttpException, Param, Query } from "@nestjs/common"; 2 | import { ApiNotFoundResponse, ApiOkResponse, ApiOperation, ApiParam, ApiQuery } from "@nestjs/swagger"; 3 | 4 | import { ApiHasPassThruAuth, Auth, RepoAuth } from "../../core"; 5 | import { GitTree } from "../../dtos"; 6 | import { ContentService } from "../../services/content"; 7 | 8 | @Controller("/repos/:remote/tree") 9 | export class TreeController { 10 | constructor(private contentService: ContentService) {} 11 | 12 | @Get([":path([^/]*)", ""]) 13 | @ApiHasPassThruAuth() 14 | @ApiOkResponse({ type: GitTree }) 15 | @ApiQuery({ name: "ref", required: false, type: "string" }) 16 | @ApiOperation({ summary: "Get tree", operationId: "tree_get" }) 17 | @ApiParam({ name: "path", type: "string" }) 18 | @ApiNotFoundResponse({}) 19 | public async getTree( 20 | @Param("remote") remote: string, 21 | @Param("path") path: string | undefined, 22 | @Query("ref") ref: string | undefined, 23 | @Auth() auth: RepoAuth, 24 | ) { 25 | const tree = await this.contentService.getContents(remote, path, ref, true, false, { auth }); 26 | if (tree instanceof HttpException) { 27 | throw tree; 28 | } 29 | return tree; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/core/pagination/paginated-response.ts: -------------------------------------------------------------------------------- 1 | import { Response } from "express"; 2 | 3 | export const TOTAL_COUNT_HEADER = "x-total-count"; 4 | 5 | export interface PaginatedList { 6 | items: T[]; 7 | page: number; 8 | perPage: number; 9 | // Total number of items 10 | total: number; 11 | } 12 | 13 | export function applyPaginatedResponse(attributes: PaginatedList, response: Response) { 14 | if (response.req) { 15 | const originalUrl = `${response.req.protocol}://${response.req.get("host")}${response.req.originalUrl}`; 16 | const nextUrl = getPageUrl(originalUrl, attributes.page + 1); 17 | const lastUrl = getPageUrl(originalUrl, Math.ceil(attributes.total / attributes.perPage)); 18 | const links = [`<${nextUrl}>; rel="next"`, `<${lastUrl}>; rel="last"`]; 19 | 20 | if (attributes.page > 1) { 21 | const prevUrl = getPageUrl(originalUrl, attributes.page - 1); 22 | links.unshift(`<${prevUrl}>; rel="prev"`); 23 | } 24 | response.setHeader("Link", links.join(", ")); 25 | response.setHeader(TOTAL_COUNT_HEADER, attributes.total); 26 | } 27 | response.send(attributes.items); 28 | } 29 | 30 | function getPageUrl(originalUrl: string, page: number) { 31 | const nextUrl = new URL(originalUrl); 32 | nextUrl.searchParams.set("page", page.toString()); 33 | return nextUrl; 34 | } 35 | -------------------------------------------------------------------------------- /src/services/branch/branch.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from "@nestjs/common"; 2 | import { Repository } from "nodegit"; 3 | 4 | import { GitBranch } from "../../dtos"; 5 | import { GitBaseOptions, RepoService } from "../repo"; 6 | 7 | @Injectable() 8 | export class BranchService { 9 | constructor(private repoService: RepoService) {} 10 | 11 | /** 12 | * List the branches 13 | * @param remote 14 | */ 15 | public async list(remote: string, options: GitBaseOptions = {}): Promise { 16 | return this.repoService.use(remote, options, async repo => { 17 | return this.listGitBranches(repo); 18 | }); 19 | } 20 | 21 | public async listGitBranches(repo: Repository): Promise { 22 | const refs = await repo.getReferences(); 23 | const branches = refs.filter(x => x.isRemote()); 24 | 25 | return Promise.all( 26 | branches.map(async ref => { 27 | const target = await ref.target(); 28 | return new GitBranch({ 29 | name: getRemoteBranchName(ref.name()), 30 | commit: { 31 | sha: target.toString(), 32 | }, 33 | }); 34 | }), 35 | ); 36 | } 37 | } 38 | 39 | export function getRemoteBranchName(refName: string) { 40 | const prefix = "refs/remotes/origin/"; 41 | return refName.slice(prefix.length); 42 | } 43 | -------------------------------------------------------------------------------- /src/core/repo-auth/repo-auth-decorators.ts: -------------------------------------------------------------------------------- 1 | import { BadRequestException, createParamDecorator } from "@nestjs/common"; 2 | import { ApiBadRequestResponse, ApiHeaders } from "@nestjs/swagger"; 3 | 4 | import { AUTH_HEADERS, RepoAuth } from "./repo-auth"; 5 | 6 | /** 7 | * Auth param decorator for controller to inject the repo auth object 8 | */ 9 | export const Auth = createParamDecorator( 10 | (_, ctx): RepoAuth => { 11 | const req = ctx.switchToHttp().getRequest(); 12 | const repoAuth = RepoAuth.fromHeaders(req.headers); 13 | if (repoAuth) { 14 | return repoAuth; 15 | } else { 16 | throw new BadRequestException("Repository authorization is malformed"); 17 | } 18 | }, 19 | ); 20 | 21 | /** 22 | * Helper to add on methods using the Auth parameter for the swagger specs to be generated correctly 23 | */ 24 | export function ApiHasPassThruAuth(): MethodDecorator { 25 | const implicitHeaders = ApiHeaders( 26 | Object.values(AUTH_HEADERS).map(header => { 27 | return { 28 | name: header, 29 | required: false, 30 | }; 31 | }), 32 | ); 33 | const badRequestResponse = ApiBadRequestResponse({ 34 | description: "When the x-authorization header is malformed", 35 | }); 36 | 37 | return (...args) => { 38 | implicitHeaders(...args); 39 | badRequestResponse(...args); 40 | }; 41 | } 42 | -------------------------------------------------------------------------------- /src/config/schema.ts: -------------------------------------------------------------------------------- 1 | import convict from "convict"; 2 | 3 | import { Configuration } from "./configuration"; 4 | 5 | export const configSchema = convict({ 6 | nodeEnv: { 7 | doc: "The code environment. This should always be production when running in production", 8 | format: ["production", "development", "test"], 9 | default: "development", 10 | env: "NODE_ENV", 11 | }, 12 | env: { 13 | doc: "Custom environment name for where the service is deployed. This is what will be used to log", 14 | format: String, 15 | default: "development", 16 | env: "APP_ENV", 17 | }, 18 | serviceName: { 19 | doc: "Name of the service. Used when uploading metrics for example", 20 | format: String, 21 | default: "git-rest-api", 22 | env: "serviceName", 23 | }, 24 | statsd: { 25 | host: { 26 | doc: "Statsd host to upload metrics using statsd", 27 | format: String, 28 | default: undefined, 29 | env: "statsd_host", 30 | }, 31 | port: { 32 | doc: "Statsd port to upload metrics using statsd", 33 | format: Number, 34 | default: undefined, 35 | env: "statsd_port", 36 | }, 37 | }, 38 | dataDir: { 39 | doc: "Temprory local data dir. Used for cloning the repos", 40 | format: String, 41 | default: "./tmp", 42 | env: "dataDir", 43 | }, 44 | }); 45 | -------------------------------------------------------------------------------- /src/services/permission/cache/permission-cache.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from "@nestjs/common"; 2 | 3 | import { RepoAuth } from "../../../core"; 4 | import { GitRemotePermission } from "../permissions"; 5 | 6 | const TOKEN_INVALIDATE_TIMEOUT = 60_000; // 60s 7 | 8 | export interface CachedPermission { 9 | lastSync: number; // Date 10 | permission: GitRemotePermission; 11 | } 12 | 13 | @Injectable() 14 | export class PermissionCacheService { 15 | private tokenPermissions = new Map(); 16 | 17 | public get(auth: RepoAuth, remote: string): GitRemotePermission | undefined { 18 | const key = this.getMapKey(auth, remote); 19 | const cache = this.tokenPermissions.get(key); 20 | if (cache === undefined) { 21 | return undefined; 22 | } 23 | const now = Date.now(); 24 | 25 | if (now - cache.lastSync > TOKEN_INVALIDATE_TIMEOUT) { 26 | this.tokenPermissions.delete(key); 27 | return undefined; 28 | } 29 | return cache.permission; 30 | } 31 | 32 | public set(auth: RepoAuth, remote: string, permission: GitRemotePermission): GitRemotePermission { 33 | const key = this.getMapKey(auth, remote); 34 | this.tokenPermissions.set(key, { permission, lastSync: Date.now() }); 35 | return permission; 36 | } 37 | 38 | private getMapKey(auth: RepoAuth, remote: string) { 39 | return `${auth.hash()}/${remote}`; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/core/logger/logger.ts: -------------------------------------------------------------------------------- 1 | import winston from "winston"; 2 | 3 | import { WINSTON_LOGGER } from "./winston-logger"; 4 | 5 | export interface LogMetadata { 6 | context?: string; 7 | [key: string]: any; 8 | } 9 | 10 | type Class = new (...args: any[]) => unknown; 11 | 12 | export class Logger { 13 | private readonly logger: winston.Logger; 14 | private readonly context: string; 15 | 16 | constructor(context: string | Class) { 17 | this.context = typeof context === "string" ? context : context.constructor.name; 18 | this.logger = WINSTON_LOGGER; 19 | } 20 | 21 | public info(message: string, meta?: LogMetadata) { 22 | this.logger.info(message, this.processMetadata(meta)); 23 | } 24 | 25 | public debug(message: string, meta?: LogMetadata) { 26 | this.logger.debug(message, this.processMetadata(meta)); 27 | } 28 | 29 | public warning(message: string, meta?: LogMetadata) { 30 | this.logger.warn(message, this.processMetadata(meta)); 31 | } 32 | 33 | public error(message: string | Error, meta?: LogMetadata) { 34 | if (message instanceof Error) { 35 | this.logger.error(message.message, this.processMetadata({ ...meta, stack: message.stack })); 36 | } else { 37 | this.logger.error(message, this.processMetadata(meta)); 38 | } 39 | } 40 | 41 | private processMetadata(meta: LogMetadata | undefined) { 42 | return { 43 | context: this.context, 44 | ...meta, 45 | }; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/core/pagination/pagination-decorators.ts: -------------------------------------------------------------------------------- 1 | import { createParamDecorator } from "@nestjs/common"; 2 | import { ApiQuery, ApiResponse } from "@nestjs/swagger"; 3 | import { Request } from "express"; 4 | 5 | import { TOTAL_COUNT_HEADER } from "./paginated-response"; 6 | import { Pagination } from "./pagination"; 7 | 8 | /** 9 | * Decorator to let swagger know about all the pagination properties available 10 | */ 11 | export function ApiPaginated(type: any): MethodDecorator { 12 | const implicitpage = ApiQuery({ name: "page", required: false, type: String }); 13 | const response = ApiResponse({ status: 200, headers: paginationHeaders, type, isArray: true }); 14 | return (...args) => { 15 | implicitpage(...args); 16 | response(...args); 17 | }; 18 | } 19 | 20 | /** 21 | * Auth param decorator for controller to inject the repo auth object 22 | */ 23 | export const Page = createParamDecorator( 24 | (_, req: Request): Pagination => { 25 | const page = parseInt(req.query.page, 10); 26 | return { 27 | page: isNaN(page) ? undefined : page, 28 | }; 29 | }, 30 | ); 31 | 32 | const paginationHeaders = { 33 | Link: { 34 | type: "string", 35 | description: 36 | "Links to navigate pagination in the format defined by [RFC 5988](https://tools.ietf.org/html/rfc5988#section-5). It will include next, last, first and prev links if applicable", 37 | }, 38 | [TOTAL_COUNT_HEADER]: { 39 | type: "integer", 40 | description: "Total count of items that can be retrieved", 41 | }, 42 | }; 43 | -------------------------------------------------------------------------------- /src/services/branch/branch.service.test.ts: -------------------------------------------------------------------------------- 1 | import { GitBranch } from "../../dtos"; 2 | import { BranchService } from "./branch.service"; 3 | 4 | const b1 = new GitBranch({ 5 | name: "master", 6 | commit: { 7 | sha: "sha1", 8 | }, 9 | }); 10 | 11 | const b2 = new GitBranch({ 12 | name: "stable", 13 | commit: { 14 | sha: "sha2", 15 | }, 16 | }); 17 | 18 | const refs = [ 19 | { isRemote: () => true, name: () => "refs/remotes/origin/master", target: () => "sha1" }, 20 | { isRemote: () => false, name: () => "refs/head/feat1", target: () => "sha3" }, 21 | { isRemote: () => true, name: () => "refs/remotes/origin/stable", target: () => "sha2" }, 22 | ]; 23 | 24 | describe("BranchService", () => { 25 | let service: BranchService; 26 | 27 | const mockRepo = { 28 | getReferences: jest.fn(() => { 29 | return refs; 30 | }), 31 | }; 32 | 33 | const repoServiceSpy = { 34 | use: jest.fn((_, _1, action) => action(mockRepo)), 35 | }; 36 | 37 | beforeEach(() => { 38 | jest.clearAllMocks(); 39 | service = new BranchService(repoServiceSpy as any); 40 | }); 41 | 42 | it("List the branches", async () => { 43 | const branches = await service.list("github.com/Azure/git-rest-api"); 44 | 45 | expect(repoServiceSpy.use).toHaveBeenCalledTimes(1); 46 | expect(repoServiceSpy.use).toHaveBeenCalledWith("github.com/Azure/git-rest-api", {}, expect.any(Function)); 47 | expect(mockRepo.getReferences).toHaveBeenCalledTimes(1); 48 | expect(branches).toEqual([b1, b2]); 49 | }); 50 | }); 51 | -------------------------------------------------------------------------------- /.ci/ci.yml: -------------------------------------------------------------------------------- 1 | trigger: 2 | - main 3 | 4 | variables: 5 | Codeql.Enabled: true 6 | 7 | jobs: 8 | - job: test 9 | displayName: Tests 10 | pool: 11 | vmImage: "ubuntu-latest" 12 | steps: 13 | - task: NodeTool@0 14 | inputs: 15 | versionSpec: "12.x" 16 | 17 | - script: npm ci 18 | displayName: "Install dependencies" 19 | 20 | - script: npm run -s test:ci 21 | displayName: "Test" 22 | 23 | - task: PublishTestResults@2 24 | inputs: 25 | testRunner: JUnit 26 | testResultsFiles: ./test-results.xml 27 | 28 | - task: PublishCodeCoverageResults@1 29 | inputs: 30 | codeCoverageTool: Cobertura 31 | summaryFileLocation: "$(System.DefaultWorkingDirectory)/**/*coverage.xml" 32 | reportDirectory: "$(System.DefaultWorkingDirectory)/**/coverage" 33 | 34 | # - script: npm run -s sdk:gen 35 | # displayName: "SDK gen" 36 | 37 | - script: npm run lint 38 | displayName: "Lint" 39 | 40 | - job: build 41 | displayName: Build and integration tests 42 | pool: 43 | vmImage: "ubuntu-latest" 44 | steps: 45 | - task: NodeTool@0 46 | inputs: 47 | versionSpec: "10.x" 48 | 49 | - script: npm ci 50 | displayName: "Install dependencies" 51 | 52 | - script: npm run build 53 | displayName: "Build" 54 | 55 | # - script: | 56 | # npm run -s start:prod & 57 | # npm run -s test:e2e 58 | # displayName: "Run integration tests" 59 | -------------------------------------------------------------------------------- /src/controllers/content/content.controller.ts: -------------------------------------------------------------------------------- 1 | import { Controller, Get, HttpException, Param, Query } from "@nestjs/common"; 2 | import { ApiNotFoundResponse, ApiOkResponse, ApiOperation, ApiParam, ApiQuery } from "@nestjs/swagger"; 3 | 4 | import { ApiHasPassThruAuth, Auth, RepoAuth } from "../../core"; 5 | import { GitContents } from "../../dtos"; 6 | import { ContentService } from "../../services/content"; 7 | import { parseBooleanFromURLParam } from "../../utils"; 8 | 9 | @Controller("/repos/:remote/contents") 10 | export class ContentController { 11 | constructor(private contentService: ContentService) {} 12 | 13 | @Get([":path([^/]*)", ""]) 14 | @ApiHasPassThruAuth() 15 | @ApiOkResponse({ type: GitContents }) 16 | @ApiQuery({ name: "ref", required: false, type: "string" }) 17 | @ApiQuery({ name: "recursive", required: false, type: "string" }) 18 | @ApiParam({ name: "path", type: "string" }) 19 | @ApiOperation({ summary: "Get content", operationId: "contents_get" }) 20 | @ApiNotFoundResponse({}) 21 | public async getContents( 22 | @Param("remote") remote: string, 23 | @Param("path") path: string | undefined, 24 | @Query("ref") ref: string | undefined, 25 | @Query("recursive") recursive: string | undefined, 26 | @Auth() auth: RepoAuth, 27 | ) { 28 | const content = await this.contentService.getContents( 29 | remote, 30 | path, 31 | ref, 32 | parseBooleanFromURLParam(recursive), 33 | true, 34 | { 35 | auth, 36 | }, 37 | ); 38 | if (content instanceof HttpException) { 39 | throw content; 40 | } 41 | return content; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /sdk/src/typescript/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "git-rest-api-sdk", 3 | "author": "Microsoft Corporation", 4 | "description": "GITRestAPI Library with typescript type definitions for node.js and browser.", 5 | "version": "0.1.0", 6 | "dependencies": { 7 | "@azure/ms-rest-js": "^1.1.0", 8 | "tslib": "^1.9.3" 9 | }, 10 | "keywords": [ 11 | "node", 12 | "azure", 13 | "git", 14 | "typescript", 15 | "browser", 16 | "isomorphic" 17 | ], 18 | "license": "MIT", 19 | "main": "./dist/bundle.js", 20 | "module": "./esm/index.js", 21 | "types": "./esm/index.d.ts", 22 | "devDependencies": { 23 | "typescript": "^3.1.1", 24 | "rollup": "^0.66.2", 25 | "rollup-plugin-node-resolve": "^3.4.0", 26 | "uglify-js": "^3.4.9" 27 | }, 28 | "homepage": "https://github.com/azure/git-rest-api", 29 | "repository": { 30 | "type": "git", 31 | "url": "https://github.com/azure/git-rest-api.git" 32 | }, 33 | "bugs": { 34 | "url": "https://github.com/azure/git-rest-api/issues" 35 | }, 36 | "files": [ 37 | "dist/**/*.js", 38 | "dist/**/*.js.map", 39 | "dist/**/*.d.ts", 40 | "dist/**/*.d.ts.map", 41 | "esm/**/*.js", 42 | "esm/**/*.js.map", 43 | "esm/**/*.d.ts", 44 | "esm/**/*.d.ts.map", 45 | "lib/**/*.ts", 46 | "rollup.config.js", 47 | "tsconfig.json" 48 | ], 49 | "scripts": { 50 | "build": "tsc && rollup -c rollup.config.js && npm run minify", 51 | "minify": "uglifyjs -c -m --comments --source-map \"content='./dist/bundle.js.map'\" -o ./dist/bundle.min.js ./dist/bundle.js", 52 | "prepack": "npm install && npm run build" 53 | }, 54 | "sideEffects": false 55 | } 56 | -------------------------------------------------------------------------------- /tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "defaultSeverity": "error", 3 | "extends": ["tslint:latest", "tslint-config-prettier", "tslint-plugin-prettier"], 4 | "jsRules": {}, 5 | "rules": { 6 | "file-name-casing": [true, "kebab-case"], 7 | "no-unused-expression": [true, "allow-fast-null-checks"], 8 | "interface-name": false, 9 | "object-literal-sort-keys": false, 10 | "no-object-literal-type-assertion": false, 11 | "no-console": true, 12 | "member-ordering": false, 13 | "no-implicit-dependencies": [true, "dev"], 14 | "prefer-conditional-expression": false, 15 | "no-submodule-imports": false, 16 | "no-default-export": true, 17 | "prettier": [true, ".prettierrc.yml"], 18 | "no-empty-interface": false, 19 | "no-floating-promises": true, // No unawaited or caught promises(This could cause process exit in later version of node) 20 | "ordered-imports": [ 21 | true, 22 | { 23 | "import-sources-order": "case-insensitive", 24 | "named-imports-order": "lowercase-last", 25 | "grouped-imports": true, 26 | "groups": [ 27 | { "match": "^\\..*$", "order": 20 }, // Have relative imports 28 | { "match": ".*", "order": 10 } 29 | ] 30 | } 31 | ], 32 | "ban": [ 33 | true, 34 | { "name": "fdescribe", "message": "Don't leave focus tests" }, 35 | { "name": "fit", "message": "Don't leave focus tests" }, 36 | { "name": "ftest", "message": "Don't leave focus tests" }, 37 | { "name": ["describe", "only"], "message": "Don't leave focus tests" }, 38 | { "name": ["it", "only"], "message": "Don't leave focus tests" }, 39 | { "name": ["test", "only"], "message": "Don't leave focus tests" } 40 | ] 41 | }, 42 | "rulesDirectory": [] 43 | } 44 | -------------------------------------------------------------------------------- /src/core/repo-auth/repo-auth.test.ts: -------------------------------------------------------------------------------- 1 | import { Cred } from "nodegit"; 2 | 3 | import { RepoAuth } from "./repo-auth"; 4 | 5 | describe("RepoAuth", () => { 6 | describe("Model", () => { 7 | it("doesn't generated any creds when no options are passed", () => { 8 | expect(new RepoAuth().toCreds()).toEqual(Cred.defaultNew()); 9 | }); 10 | 11 | it("returns undefined if invalid basic header", () => { 12 | expect(RepoAuth.fromHeaders({ "x-authorization": "Basi invlid" })).toBeUndefined(); 13 | }); 14 | 15 | it("doesn't generate some basic password creds when using oath token", () => { 16 | expect(RepoAuth.fromHeaders({ "x-github-token": "token-1" })!.toCreds()).toEqual( 17 | Cred.userpassPlaintextNew("token-1", "x-oauth-basic"), 18 | ); 19 | }); 20 | 21 | it("should return different hashes per cred", () => { 22 | const hash1 = new RepoAuth({ username: "foo", password: "bar" }).hash(); 23 | const hash2 = new RepoAuth({ username: "foo2", password: "bar" }).hash(); 24 | const hash3 = new RepoAuth({ username: "foo", password: "bar2" }).hash(); 25 | 26 | expect(hash1).not.toBe(hash2); 27 | expect(hash2).not.toBe(hash3); 28 | expect(hash1).not.toBe(hash3); 29 | 30 | expect(hash1).toBe( 31 | "6bd974a6ce1305174fff5df0cefd2025005861ad19332177290f10af00e0b5dee8e563417bc6f509e50527d35ec48986593d347cfc41069e8bfad24e70c5eabb", 32 | ); 33 | expect(hash2).toBe( 34 | "cd6f6487398b42e04b68676f8567d4c9494f49347862272ebe7cba3ce5c9fa2c05cdc3ee302ae42bbb00632e7ad78f1b0ecd1b37634ffcae3756b427e7c8080b", 35 | ); 36 | expect(hash3).toBe( 37 | "34f5826bdf4f7863ede840bfc394a585c233862eee7d6c6dcdc0287b98626fae43f91f41700144c22207372b24ecdec35f6b1d4ebd882d2a5b7b25918b3c18aa", 38 | ); 39 | }); 40 | }); 41 | }); 42 | -------------------------------------------------------------------------------- /src/services/disk-usage/disk-usage.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from "@nestjs/common"; 2 | import diskusage from "diskusage"; 3 | import { Observable, Subscription, from, timer } from "rxjs"; 4 | import { filter, map, publishReplay, refCount, switchMap } from "rxjs/operators"; 5 | 6 | import { Configuration } from "../../config"; 7 | import { Logger, Telemetry } from "../../core"; 8 | 9 | const DISK_USAGE_COLLECTION_INTERVAL = 10_000; 10 | 11 | export interface DiskUsage { 12 | available: number; 13 | total: number; 14 | used: number; 15 | } 16 | 17 | @Injectable() 18 | export class DiskUsageService { 19 | public dataDiskUsage: Observable; 20 | 21 | private logger = new Logger("DiskUsageService"); 22 | 23 | constructor(private config: Configuration, private telemetry: Telemetry) { 24 | this.dataDiskUsage = timer(0, DISK_USAGE_COLLECTION_INTERVAL).pipe( 25 | switchMap(() => from(this.checkDataDiskUsage())), 26 | filter((x: T | undefined): x is T => x !== undefined), 27 | map(({ available, total }) => { 28 | return { available, total, used: total - available }; 29 | }), 30 | publishReplay(1), 31 | refCount(), 32 | ); 33 | } 34 | 35 | public startCollection(): Subscription { 36 | return this.dataDiskUsage.subscribe(usage => { 37 | this.emitDiskUsage(usage); 38 | }); 39 | } 40 | 41 | private emitDiskUsage(usage: DiskUsage) { 42 | this.telemetry.emitMetric({ 43 | name: "DATA_DISK_USAGE_AVAILABLE", 44 | value: usage.available, 45 | }); 46 | this.logger.debug("Data disk usage", usage); 47 | } 48 | 49 | private async checkDataDiskUsage(): Promise { 50 | try { 51 | return await diskusage.check(this.config.dataDir); 52 | } catch (error) { 53 | this.logger.error("Failed to get disk usage", error as any); 54 | return undefined; 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/scripts/swagger-generation.ts: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | import { promisify } from "util"; 3 | 4 | import { createApp } from "../app"; 5 | 6 | const writeFile = promisify(fs.writeFile); 7 | const readFile = promisify(fs.readFile); 8 | 9 | const SWAGGER_FILE_PATH = "./swagger-spec.json"; 10 | 11 | export async function generateSwagger(): Promise { 12 | const { document } = await createApp(); 13 | reorderPaths(document.paths); 14 | 15 | return JSON.stringify(document, null, 2); 16 | } 17 | 18 | export async function saveSwagger() { 19 | const specs = await generateSwagger(); 20 | await writeFile(SWAGGER_FILE_PATH, specs); 21 | } 22 | 23 | export async function getSavedSwagger(): Promise { 24 | const response = await readFile(SWAGGER_FILE_PATH); 25 | return response.toString(); 26 | } 27 | 28 | /** 29 | * Because of how the nestjs generate the path params they might not be in order which could lead to breaking changes without noticing. 30 | * Force the params to be ordered how they are defined in the path 31 | */ 32 | export function reorderPaths(paths: StringMap>) { 33 | for (const [path, methods] of Object.entries(paths)) { 34 | for (const def of Object.values(methods)) { 35 | if (def.parameters) { 36 | def.parameters = getOrderedPathParams(path, def.parameters); 37 | } 38 | } 39 | } 40 | } 41 | 42 | interface SwaggerParam { 43 | type: string; 44 | name: string; 45 | required: boolean; 46 | in: "path" | "header"; 47 | } 48 | 49 | export function getOrderedPathParams(path: string, parameters: SwaggerParam[]) { 50 | const pathParams = parameters.filter(x => x.in === "path"); 51 | const otherParams = parameters.filter(x => x.in !== "path"); 52 | 53 | const sortedPathParams = [...pathParams].sort((a, b) => { 54 | return path.indexOf(`{${a.name}}`) - path.indexOf(`{${b.name}}`); 55 | }); 56 | return [...sortedPathParams, ...otherParams]; 57 | } 58 | -------------------------------------------------------------------------------- /src/config/configuration.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from "@nestjs/common"; 2 | 3 | import { developmentConfig } from "./development"; 4 | import { productionConfig } from "./production"; 5 | import { configSchema } from "./schema"; 6 | import { testConfig } from "./test"; 7 | 8 | export type NodeEnv = "production" | "development" | "test"; 9 | 10 | export interface StatsdConfig { 11 | readonly host: string | undefined; 12 | readonly port: number | undefined; 13 | } 14 | 15 | @Injectable() 16 | export class Configuration { 17 | /** 18 | * Correspond the env this code is running on. 19 | * If not deveopping or running test it should always be "production". 20 | * This should be used to decide to optimize the code for dev or prod. 21 | */ 22 | public readonly nodeEnv: NodeEnv; 23 | 24 | /** 25 | * This is the actual environment name where the service is run.(Stagging, PPE, Prod, etc.) 26 | * This can be anything. 27 | */ 28 | public readonly env: string; 29 | public readonly serviceName: string; 30 | public readonly statsd: StatsdConfig; 31 | 32 | /** 33 | * Path to handle data(e.g. CLone repos) 34 | * This dir will be cleared. 35 | */ 36 | public readonly dataDir: string; 37 | 38 | constructor() { 39 | const environmentOverrides: Record> = { 40 | production: productionConfig, 41 | development: developmentConfig, 42 | test: testConfig, 43 | }; 44 | 45 | const nodeEnv: NodeEnv = configSchema.get("nodeEnv"); 46 | 47 | // Load environment dependent configuration 48 | configSchema.load(environmentOverrides[nodeEnv]); 49 | 50 | // Perform validation 51 | configSchema.validate({ allowed: "strict" }); 52 | 53 | this.nodeEnv = nodeEnv; 54 | this.env = configSchema.get("env"); 55 | this.serviceName = configSchema.get("serviceName"); 56 | this.statsd = configSchema.get("statsd"); 57 | this.dataDir = configSchema.get("dataDir"); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/services/repo/mutex/mutex.test.ts: -------------------------------------------------------------------------------- 1 | import { delay } from "../../../utils"; 2 | import { Mutex } from "./mutex"; 3 | 4 | describe("Mutex", () => { 5 | let mutex: Mutex; 6 | 7 | beforeEach(() => { 8 | mutex = new Mutex(); 9 | }); 10 | 11 | it("acquire multiple read locks", async () => { 12 | const lock1 = await mutex.lock(); 13 | const lock2 = await mutex.lock(); 14 | expect(lock1.id).not.toEqual(lock2.id); 15 | lock1.release(); 16 | lock2.release(); 17 | }); 18 | 19 | it("acquire exclusive locks sequentially", async () => { 20 | const lock1Spy = jest.fn(); 21 | const lock2Spy = jest.fn(); 22 | const lock1Promise = mutex.lock({ exclusive: true }).then(x => { 23 | lock1Spy(); 24 | return x; 25 | }); 26 | const lock2Promise = mutex.lock({ exclusive: true }).then(x => { 27 | lock2Spy(); 28 | return x; 29 | }); 30 | 31 | await delay(); 32 | 33 | expect(lock1Spy).toHaveBeenCalledTimes(1); 34 | expect(lock2Spy).not.toHaveBeenCalled(); 35 | 36 | const lock1 = await lock1Promise; 37 | lock1.release(); 38 | await delay(); 39 | 40 | expect(lock2Spy).toHaveBeenCalledTimes(1); 41 | 42 | const lock2 = await lock2Promise; 43 | lock2.release(); 44 | }); 45 | 46 | it("acquire exclusive wait on shared lock", async () => { 47 | const lock1Spy = jest.fn(); 48 | const lock2Spy = jest.fn(); 49 | const lock1Promise = mutex.lock().then(x => { 50 | lock1Spy(); 51 | return x; 52 | }); 53 | const lock2Promise = mutex.lock({ exclusive: true }).then(x => { 54 | lock2Spy(); 55 | return x; 56 | }); 57 | 58 | await delay(); 59 | 60 | expect(lock1Spy).toHaveBeenCalledTimes(1); 61 | expect(lock2Spy).not.toHaveBeenCalled(); 62 | 63 | const lock1 = await lock1Promise; 64 | lock1.release(); 65 | await delay(); 66 | 67 | expect(lock2Spy).toHaveBeenCalledTimes(1); 68 | 69 | const lock2 = await lock2Promise; 70 | lock2.release(); 71 | }); 72 | }); 73 | -------------------------------------------------------------------------------- /src/controllers/commits/commits.controller.ts: -------------------------------------------------------------------------------- 1 | import { Controller, Get, NotFoundException, Param, Query, Res } from "@nestjs/common"; 2 | import { ApiNotFoundResponse, ApiOkResponse, ApiOperation, ApiQuery } from "@nestjs/swagger"; 3 | import { Response } from "express"; 4 | 5 | import { ApiHasPassThruAuth, Auth, RepoAuth } from "../../core"; 6 | import { ApiPaginated, Page, Pagination, applyPaginatedResponse } from "../../core/pagination"; 7 | import { GitCommit } from "../../dtos"; 8 | import { CommitService } from "../../services"; 9 | 10 | @Controller("/repos/:remote/commits") 11 | export class CommitsController { 12 | constructor(private commitService: CommitService) {} 13 | 14 | @Get() 15 | @ApiHasPassThruAuth() 16 | @ApiNotFoundResponse({}) 17 | @ApiOperation({ summary: "List commits", operationId: "commits_list" }) 18 | @ApiQuery({ 19 | name: "ref", 20 | required: false, 21 | description: "Reference to list the commits from. Can be a branch or a commit. Default to master", 22 | type: String, 23 | }) 24 | @ApiPaginated(GitCommit) 25 | public async list( 26 | @Param("remote") remote: string, 27 | @Query("ref") ref: string | undefined, 28 | @Auth() auth: RepoAuth, 29 | @Page() pagination: Pagination, 30 | @Res() response: Response, 31 | ) { 32 | const commits = await this.commitService.list(remote, { auth, ref, pagination }); 33 | if (commits instanceof NotFoundException) { 34 | throw commits; 35 | } 36 | return applyPaginatedResponse(commits, response); 37 | } 38 | 39 | @Get(":commitSha") 40 | @ApiHasPassThruAuth() 41 | @ApiOkResponse({ type: GitCommit }) 42 | @ApiNotFoundResponse({}) 43 | @ApiOperation({ summary: "Get a commit", operationId: "commits_get" }) 44 | public async get(@Param("remote") remote: string, @Param("commitSha") commitSha: string, @Auth() auth: RepoAuth) { 45 | const commit = await this.commitService.get(remote, commitSha, { auth }); 46 | 47 | if (!commit) { 48 | throw new NotFoundException(`Commit with sha ${commitSha} was not found`); 49 | } 50 | return commit; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/services/repo-index/repo-index.service.test.ts: -------------------------------------------------------------------------------- 1 | import { RepoReferenceRecord } from "../../models"; 2 | import { RepoIndexService } from "./repo-index.service"; 3 | 4 | describe("RepoIndexService", () => { 5 | let service: RepoIndexService; 6 | let now: number; 7 | const nowSpy = jest.fn(() => now); 8 | let originalNow: typeof Date.now; 9 | 10 | const dbRepoSpy = {}; 11 | const connectionSpy = { 12 | getRepository: jest.fn(() => dbRepoSpy), 13 | }; 14 | beforeEach(() => { 15 | originalNow = Date.now; 16 | now = Date.now(); 17 | Date.now = nowSpy; 18 | service = new RepoIndexService(connectionSpy as any); 19 | }); 20 | 21 | afterEach(() => { 22 | Date.now = originalNow; 23 | }); 24 | 25 | it("Calls the repo", () => { 26 | expect(connectionSpy.getRepository).toHaveBeenCalledTimes(1); 27 | expect(connectionSpy.getRepository).toHaveBeenCalledWith(RepoReferenceRecord); 28 | }); 29 | 30 | it("get the least recently used repos", () => { 31 | service.markRepoAsOpened("foo-1"); 32 | now += 100; 33 | service.markRepoAsFetched("foo-2"); 34 | now += 100; 35 | service.markRepoAsFetched("foo-3"); 36 | now += 100; 37 | service.markRepoAsOpened("foo-4"); 38 | now -= 10000; 39 | service.markRepoAsOpened("foo-0"); 40 | 41 | expect(service.getLeastUsedRepos(3)).toEqual(["foo-0", "foo-1", "foo-2"]); 42 | }); 43 | 44 | it("needs to fetch if the repo was never opened", () => { 45 | expect(service.needToFetch("foo-1")).toBe(true); 46 | }); 47 | 48 | it("needs to fetch if the repo was never fetched", () => { 49 | service.markRepoAsOpened("foo-1"); 50 | 51 | expect(service.needToFetch("foo-1")).toBe(true); 52 | }); 53 | 54 | it("needs to fetch only after the cache timeout expire", () => { 55 | service.markRepoAsFetched("foo-1"); 56 | expect(service.needToFetch("foo-1")).toBe(false); 57 | 58 | now += 29_999; 59 | expect(service.needToFetch("foo-1")).toBe(false); 60 | now += 2; 61 | expect(service.needToFetch("foo-1")).toBe(true); 62 | now += 10_000; 63 | expect(service.needToFetch("foo-1")).toBe(true); 64 | }); 65 | }); 66 | -------------------------------------------------------------------------------- /.ci/publish.yml: -------------------------------------------------------------------------------- 1 | trigger: 2 | - master 3 | 4 | variables: 5 | Codeql.Enabled: true 6 | 7 | pr: none 8 | 9 | stages: 10 | # - stage: Build_Docker 11 | # jobs: 12 | # - job: Build_Docker 13 | # displayName: Build and publish to hub.docker.com 14 | # pool: 15 | # vmImage: "Ubuntu-16.04" 16 | # variables: 17 | # build_counter: $[counter('versioncounter', 1)] 18 | # steps: 19 | # - script: | 20 | # version=$(npm run -s get-version) 21 | # echo "Current package version is $version" 22 | # echo "Current build number is $(build_counter)" 23 | 24 | # echo "##vso[build.updatebuildnumber]$version.$(build_counter)" 25 | # echo "##vso[task.setvariable variable=NpmVersion]$version" 26 | # displayName: Resolve package version 27 | # - task: Docker@2 28 | # displayName: "Build azuredevx/git-rest-api" 29 | # inputs: 30 | # containerRegistry: "git-rest-api docker" 31 | # repository: "azuredevx/git-rest-api" 32 | # tags: | 33 | # $(Build.BuildNumber) 34 | # $(NpmVersion)-latest 35 | # latest 36 | 37 | - stage: build_sdks 38 | displayName: Build SDKs 39 | dependsOn: [] 40 | jobs: 41 | - job: build 42 | pool: 43 | vmImage: "ubuntu-latest" 44 | steps: 45 | - script: npm ci 46 | displayName: Install 47 | - script: npm run sdk:gen 48 | displayName: Generate SDKs 49 | - script: npm pack 50 | workingDirectory: sdk/out/typescript 51 | displayName: Pack 52 | - task: CopyFiles@2 53 | displayName: "Copy Files to: drop" 54 | inputs: 55 | sourceFolder: ./sdk/out/typescript 56 | contents: "*.tgz" 57 | targetFolder: $(Build.ArtifactStagingDirectory)/drop 58 | - task: PublishPipelineArtifact@0 59 | inputs: 60 | artifactName: "drop" 61 | targetPath: $(Build.ArtifactStagingDirectory)/drop 62 | -------------------------------------------------------------------------------- /src/services/permission/cache/permission-cache.service.test.ts: -------------------------------------------------------------------------------- 1 | import { RepoAuth } from "../../../core"; 2 | import { GitRemotePermission } from "../permissions"; 3 | import { PermissionCacheService } from "./permission-cache.service"; 4 | 5 | const remote = "github.com/Azure/git-rest-specs"; 6 | 7 | const auth = new RepoAuth({ username: "token-1", password: "x-oauth-token" }); 8 | describe("PermissionService", () => { 9 | let service: PermissionCacheService; 10 | 11 | beforeEach(() => { 12 | jest.clearAllTimers(); 13 | jest.useFakeTimers(); 14 | service = new PermissionCacheService(); 15 | }); 16 | 17 | it("returns undefined when permission has not been set", () => { 18 | expect(service.get(auth, remote)).toBeUndefined(); 19 | }); 20 | 21 | it("returns none when permission is set to none", () => { 22 | service.set(auth, remote, GitRemotePermission.None); 23 | expect(service.get(auth, remote)).toBe(GitRemotePermission.None); 24 | }); 25 | 26 | it("set a permission", () => { 27 | service.set(auth, remote, GitRemotePermission.Read); 28 | expect(service.get(auth, remote)).toBe(GitRemotePermission.Read); 29 | }); 30 | 31 | it("clear permission after a timeout", () => { 32 | const now = Date.now(); 33 | 34 | service.set(auth, remote, GitRemotePermission.Read); 35 | expect(service.get(auth, remote)).toBe(GitRemotePermission.Read); 36 | mockDate(now + 50_000); 37 | // Should still be cached 38 | expect(service.get(auth, remote)).toBe(GitRemotePermission.Read); 39 | 40 | mockDate(now + 60_010); 41 | expect(service.get(auth, remote)).toBeUndefined(); 42 | }); 43 | 44 | it("doesn't get permission from another key", () => { 45 | service.set(auth, remote, GitRemotePermission.Read); 46 | expect(service.get(new RepoAuth(), remote)).toBeUndefined(); 47 | expect(service.get(new RepoAuth({ username: "foo", password: "x-oauth-token" }), remote)).toBeUndefined(); 48 | expect(service.get(new RepoAuth({ username: "token-1", password: "x-other-token" }), remote)).toBeUndefined(); 49 | }); 50 | }); 51 | 52 | function mockDate(now: number) { 53 | jest.spyOn(global.Date, "now").mockImplementation(() => now); 54 | } 55 | -------------------------------------------------------------------------------- /src/services/repo/mutex/mutex.ts: -------------------------------------------------------------------------------- 1 | import uuid from "uuid/v4"; 2 | 3 | export interface Lock { 4 | readonly id: string; 5 | readonly release: () => void; 6 | } 7 | 8 | export interface LockOptions { 9 | exclusive: boolean; 10 | } 11 | 12 | /** 13 | * Mutex supporting shared and exclusive locks. 14 | * Exclusive locks have priority: If one is requested, following shared lock request will have to wait for the exclusive lock to be acquited and released 15 | */ 16 | export class Mutex { 17 | public get pending() { 18 | return !( 19 | this.sharedLocks.size === 0 && 20 | this.exclusiveLocks.size === 0 && 21 | this.sharedQueue.length === 0 && 22 | this.exclusiveQueue.length === 0 23 | ); 24 | } 25 | 26 | private readonly sharedLocks = new Set(); 27 | private readonly exclusiveLocks = new Set(); 28 | private readonly exclusiveQueue: Array<(lock: Lock) => void> = []; 29 | private readonly sharedQueue: Array<(lock: Lock) => void> = []; 30 | 31 | public lock({ exclusive }: LockOptions = { exclusive: false }): Promise { 32 | const promise = new Promise(resolve => 33 | exclusive ? this.exclusiveQueue.push(resolve) : this.sharedQueue.push(resolve), 34 | ); 35 | this._dispatchNext(); 36 | 37 | return promise; 38 | } 39 | 40 | private _dispatchNext() { 41 | if (this.exclusiveQueue.length > 0) { 42 | if (this.sharedLocks.size === 0 && this.exclusiveLocks.size === 0) { 43 | const resolve = this.exclusiveQueue.shift(); 44 | if (resolve) { 45 | const lock = this.createLock(this.exclusiveLocks); 46 | resolve(lock); 47 | } 48 | } 49 | } else if (this.sharedQueue.length > 0) { 50 | if (this.exclusiveLocks.size === 0) { 51 | const resolve = this.sharedQueue.shift(); 52 | if (resolve) { 53 | const lock = this.createLock(this.sharedLocks); 54 | resolve(lock); 55 | } 56 | } 57 | } 58 | } 59 | 60 | private createLock(set: Set) { 61 | const id = uuid(); 62 | set.add(id); 63 | 64 | return { 65 | id, 66 | release: () => { 67 | set.delete(id); 68 | this._dispatchNext(); 69 | }, 70 | }; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/services/repo-cleanup/repo-cleanup.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from "@nestjs/common"; 2 | import { from, of } from "rxjs"; 3 | import { catchError, delay, exhaustMap, filter } from "rxjs/operators"; 4 | 5 | import { Logger } from "../../core"; 6 | import { DiskUsageService } from "../disk-usage"; 7 | import { RepoService } from "../repo"; 8 | import { RepoIndexService } from "../repo-index"; 9 | 10 | /** 11 | * Service handling cleanup of the disk where repos are cloned when available space is getting low. 12 | */ 13 | @Injectable() 14 | export class RepoCleanupService { 15 | private logger = new Logger(RepoCleanupService); 16 | 17 | constructor( 18 | private repoIndexService: RepoIndexService, 19 | private repoService: RepoService, 20 | private diskUsageService: DiskUsageService, 21 | ) {} 22 | 23 | public start() { 24 | return this.diskUsageService.dataDiskUsage 25 | .pipe( 26 | filter(x => { 27 | const freeRatio = x.available / x.total; 28 | return freeRatio < 0.5; 29 | }), 30 | exhaustMap(() => { 31 | const count = this.getNumberOfReposToRemove(); 32 | if (this.repoIndexService.size === 0) { 33 | this.logger.error("There isn't any repo cached on disk. Space is most likely used by something else."); 34 | } 35 | const total = this.repoIndexService.size; 36 | this.logger.warning( 37 | `Disk availability is low. Removing least recently used repos. Total repos: ${total}, Removing: ${count}`, 38 | ); 39 | const repos = this.repoIndexService.getLeastUsedRepos(count); 40 | return from(Promise.all(repos.map(x => this.repoService.deleteLocalRepo(x)))).pipe( 41 | delay(2000), 42 | catchError(error => { 43 | this.logger.error("Error occured when deleting repos", { error }); 44 | return of(undefined); 45 | }), 46 | ); 47 | }), 48 | ) 49 | .subscribe({ 50 | error: error => { 51 | this.logger.error("Error occured in repo cleanup", { error }); 52 | }, 53 | }); 54 | } 55 | 56 | private getNumberOfReposToRemove() { 57 | const count = Math.ceil(this.repoIndexService.size / 100); 58 | return Math.max(Math.min(count, 10), 1); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/app.module.ts: -------------------------------------------------------------------------------- 1 | import { MiddlewareConsumer, Module, NestModule } from "@nestjs/common"; 2 | import { APP_INTERCEPTOR } from "@nestjs/core"; 3 | import { Connection } from "typeorm"; 4 | 5 | import { Configuration } from "./config"; 6 | import { 7 | BranchesController, 8 | CommitsController, 9 | CompareController, 10 | ContentController, 11 | HealthCheckController, 12 | } from "./controllers"; 13 | import { TreeController } from "./controllers/tree/tree.controller"; 14 | import { Telemetry, createTelemetry } from "./core"; 15 | import { ContextMiddleware, LoggingInterceptor } from "./middlewares"; 16 | import { 17 | AppService, 18 | BranchService, 19 | CommitService, 20 | CompareService, 21 | ContentService, 22 | DiskUsageService, 23 | FSService, 24 | HttpService, 25 | PermissionCacheService, 26 | PermissionService, 27 | RepoCleanupService, 28 | RepoService, 29 | createDBConnection, 30 | } from "./services"; 31 | import { RepoIndexService } from "./services/repo-index"; 32 | 33 | @Module({ 34 | imports: [], 35 | controllers: [ 36 | HealthCheckController, 37 | BranchesController, 38 | CommitsController, 39 | CompareController, 40 | ContentController, 41 | TreeController, 42 | ], 43 | providers: [ 44 | AppService, 45 | CompareService, 46 | RepoService, 47 | FSService, 48 | BranchService, 49 | PermissionService, 50 | PermissionCacheService, 51 | HttpService, 52 | Configuration, 53 | CommitService, 54 | ContentService, 55 | DiskUsageService, 56 | RepoIndexService, 57 | RepoCleanupService, 58 | { 59 | provide: APP_INTERCEPTOR, 60 | useClass: LoggingInterceptor, 61 | }, 62 | { 63 | provide: Telemetry, 64 | useFactory: (config: Configuration) => createTelemetry(config), 65 | inject: [Configuration], 66 | }, 67 | { 68 | provide: Connection, 69 | useFactory: (config: Configuration) => createDBConnection(config), 70 | inject: [Configuration], 71 | }, 72 | ], 73 | }) 74 | export class AppModule implements NestModule { 75 | constructor(private diskUsage: DiskUsageService, private repoCleanupService: RepoCleanupService) {} 76 | public configure(consumer: MiddlewareConsumer) { 77 | this.diskUsage.startCollection(); 78 | this.repoCleanupService.start(); 79 | 80 | consumer.apply(ContextMiddleware).forRoutes("*"); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/services/permission/permission.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from "@nestjs/common"; 2 | 3 | import { RepoAuth } from "../../core"; 4 | import { RepoUtils } from "../../utils"; 5 | import { HttpService } from "../http"; 6 | import { PermissionCacheService } from "./cache"; 7 | import { GitRemotePermission } from "./permissions"; 8 | 9 | @Injectable() 10 | export class PermissionService { 11 | constructor(private cache: PermissionCacheService, private http: HttpService) {} 12 | 13 | public async get(auth: RepoAuth, remote: string): Promise { 14 | const cached = this.cache.get(auth, remote); 15 | if (cached !== undefined) { 16 | return cached; 17 | } 18 | return this.retrievePermissions(auth, remote); 19 | } 20 | 21 | public set(auth: RepoAuth, remote: string, permission: GitRemotePermission) { 22 | return this.cache.set(auth, remote, permission); 23 | } 24 | 25 | private async retrievePermissions(auth: RepoAuth, remote: string) { 26 | const gitUrl = RepoUtils.getUrl(remote); 27 | const canWrite = await this.checkWritePermission(auth, gitUrl); 28 | if (canWrite) { 29 | return this.set(auth, remote, GitRemotePermission.Write); 30 | } 31 | 32 | const canRead = await this.checkReadPermission(auth, gitUrl); 33 | if (canRead) { 34 | return this.set(auth, remote, GitRemotePermission.Read); 35 | } 36 | return this.set(auth, remote, GitRemotePermission.None); 37 | } 38 | 39 | private async checkWritePermission(auth: RepoAuth, gitUrl: string): Promise { 40 | const response = await this.http.fetch(`${gitUrl}/${GitServices.Push}`, { 41 | headers: this.getHeaders(auth), 42 | method: "POST", 43 | }); 44 | return response.status === 200; 45 | } 46 | 47 | private async checkReadPermission(auth: RepoAuth, gitUrl: string): Promise { 48 | const response = await this.http.fetch(`${gitUrl}/${GitServices.Pull}`, { 49 | headers: this.getHeaders(auth), 50 | method: "POST", 51 | }); 52 | return response.status === 200; 53 | } 54 | 55 | private getHeaders(auth: RepoAuth): StringMap { 56 | const header = auth.toAuthorizationHeader(); 57 | if (!header) { 58 | return {}; 59 | } 60 | return { 61 | Authorization: header, 62 | }; 63 | } 64 | } 65 | 66 | enum GitServices { 67 | Push = "git-receive-pack", 68 | Pull = "git-upload-pack", 69 | } 70 | -------------------------------------------------------------------------------- /src/controllers/branches/branches.controller.e2e.ts: -------------------------------------------------------------------------------- 1 | import { TEST_REPO, UNENCODED_TEST_REPO, deleteLocalRepo, e2eClient } from "../../../test/e2e"; 2 | import { delay } from "../../utils"; 3 | 4 | describe("Test branch controller", () => { 5 | it("doesn't conflict when getting the same repo twice at the same time", async () => { 6 | await delay(2000); // Need to wait for the server to release locks on the repo from previous tests. As e2e test are run in squence this should be a fixed amount of time 7 | await deleteLocalRepo(UNENCODED_TEST_REPO); 8 | await delay(100); 9 | const responses = await Promise.all([ 10 | e2eClient.fetch(`/repos/${TEST_REPO}/branches`), 11 | // Delay a little to start the clone 12 | delay(10).then(() => e2eClient.fetch(`/repos/${TEST_REPO}/branches`)), 13 | delay(20).then(() => e2eClient.fetch(`/repos/${TEST_REPO}/branches`)), 14 | delay(30).then(() => e2eClient.fetch(`/repos/${TEST_REPO}/branches`)), 15 | delay(40).then(() => e2eClient.fetch(`/repos/${TEST_REPO}/branches`)), 16 | delay(50).then(() => e2eClient.fetch(`/repos/${TEST_REPO}/branches`)), 17 | ]); 18 | 19 | for (const response of responses) { 20 | expect(response.status).toEqual(200); 21 | } 22 | 23 | const [first, ...others] = await Promise.all(responses.map(x => x.json() as Promise)); 24 | for (const content of others) { 25 | expect(sortBy(content, x => x.name)).toEqual(sortBy(first, x => x.name)); 26 | } 27 | 28 | // Make sure the server didn't crash. This is a regression test where a segfault happened freeing the Repository object. 29 | await delay(1000); 30 | const lastResponse = await e2eClient.fetch(`/repos/${TEST_REPO}/branches`); 31 | 32 | expect(lastResponse.status).toEqual(200); 33 | }); 34 | 35 | it("List branches in the test repo", async () => { 36 | const response = await e2eClient.fetch(`/repos/${TEST_REPO}/branches`); 37 | expect(response.status).toEqual(200); 38 | const content: any[] = await response.json(); 39 | expect(sortBy(content, x => x.name)).toMatchPayload("branches_list"); 40 | }); 41 | }); 42 | 43 | function sortBy(array: T[], attr: (item: T) => any): T[] { 44 | return array.sort((a, b) => { 45 | const aAttr = attr(a); 46 | const bAttr = attr(b); 47 | 48 | if (aAttr < bAttr) { 49 | return -1; 50 | } else if (aAttr > bAttr) { 51 | return 1; 52 | } else { 53 | return 0; 54 | } 55 | }); 56 | } 57 | -------------------------------------------------------------------------------- /src/middlewares/context/context.middleware.test.ts: -------------------------------------------------------------------------------- 1 | import { getContext, setContext } from "../../core"; 2 | import { ContextMiddleware } from "./context.middleware"; 3 | 4 | describe("ContextMiddleware", () => { 5 | let middleware: ContextMiddleware; 6 | const response = { 7 | setHeader: jest.fn(), 8 | }; 9 | let request: { requestId?: string }; 10 | beforeEach(() => { 11 | jest.clearAllMocks(); 12 | request = {}; 13 | middleware = new ContextMiddleware(); 14 | }); 15 | 16 | describe("Setting some context", () => { 17 | it("should have set a request id", () => { 18 | middleware.use(request as any, response as any, async () => { 19 | expect(request.requestId).not.toBeUndefined(); 20 | 21 | expect(response.setHeader).toHaveBeenCalledTimes(1); 22 | expect(response.setHeader).toHaveBeenCalledWith("x-request-id", request.requestId); 23 | }); 24 | }); 25 | }); 26 | 27 | it("keeps the context with async await", done => { 28 | middleware.use(request as any, response as any, async () => { 29 | setContext("requestId", "req-1"); 30 | expect(getContext("requestId")).toEqual("req-1"); 31 | 32 | await Promise.resolve(); 33 | expect(getContext("requestId")).toEqual("req-1"); 34 | done(); 35 | }); 36 | }); 37 | 38 | it("keeps the context with callbacks", done => { 39 | middleware.use(request as any, response as any, async () => { 40 | setContext("requestId", "req-1"); 41 | expect(getContext("requestId")).toEqual("req-1"); 42 | 43 | setTimeout(() => { 44 | expect(getContext("requestId")).toEqual("req-1"); 45 | done(); 46 | }); 47 | }); 48 | }); 49 | 50 | it("doesn't mix up the contexts. Each request gets its own", done => { 51 | middleware.use(request as any, response as any, async () => { 52 | setContext("requestId", "req-1"); 53 | expect(getContext("requestId")).toEqual("req-1"); 54 | 55 | setTimeout(() => { 56 | expect(getContext("requestId")).toEqual("req-1"); 57 | setTimeout(() => { 58 | expect(getContext("requestId")).toEqual("req-1"); 59 | done(); 60 | }); 61 | }); 62 | }); 63 | 64 | middleware.use(request as any, response as any, async () => { 65 | setContext("requestId", "req-2"); 66 | expect(getContext("requestId")).toEqual("req-2"); 67 | 68 | setTimeout(() => { 69 | expect(getContext("requestId")).toEqual("req-2"); 70 | done(); 71 | }); 72 | }); 73 | }); 74 | }); 75 | -------------------------------------------------------------------------------- /src/core/logger/winston-logger.ts: -------------------------------------------------------------------------------- 1 | import chalk from "chalk"; 2 | import jsonStringify from "fast-safe-stringify"; 3 | import { MESSAGE } from "triple-beam"; 4 | import winston, { LoggerOptions, format } from "winston"; 5 | import winstonDailyFile from "winston-daily-rotate-file"; 6 | 7 | import { Configuration } from "../../config"; 8 | import { getContext } from "../context"; 9 | 10 | // Adds default metadata to logs 11 | const addProdMetadata = winston.format(info => { 12 | info.Env = config.env; 13 | info.Service = config.serviceName; 14 | if (!info.requestId) { 15 | info.requestId = getContext("requestId"); 16 | } 17 | return info; 18 | }); 19 | 20 | const consoleTransport: winston.transports.ConsoleTransportOptions = { 21 | handleExceptions: true, 22 | level: "info", 23 | }; 24 | 25 | const customFormat = format(info => { 26 | const { message, level, timestamp, context, trace, ...others } = info; 27 | const stringifiedRest = jsonStringify(others); 28 | 29 | const padding = (info.padding && info.padding[level]) || ""; 30 | const coloredTime = chalk.dim.yellow.bold(timestamp); 31 | const coloredContext = chalk.grey(context); 32 | let coloredMessage = `${level}:${padding} ${coloredTime} | [${coloredContext}] ${message}`; 33 | if (stringifiedRest !== "{}") { 34 | coloredMessage = `${coloredMessage} ${stringifiedRest}`; 35 | } 36 | if (trace) { 37 | coloredMessage = `${coloredMessage}\n${trace}`; 38 | } 39 | info[MESSAGE] = coloredMessage; 40 | return info; 41 | }); 42 | 43 | const config = new Configuration(); 44 | 45 | // In development only we want to have the logs printed nicely. For production we want json log lines that can be parsed easily 46 | if (config.nodeEnv === "development") { 47 | consoleTransport.format = winston.format.combine( 48 | winston.format.timestamp({ 49 | format: "YYYY-MM-DD HH:mm:ss", 50 | }), 51 | winston.format.colorize(), 52 | customFormat(), 53 | ); 54 | } else { 55 | consoleTransport.format = winston.format.combine( 56 | addProdMetadata(), 57 | winston.format.timestamp(), 58 | winston.format.json(), 59 | ); 60 | } 61 | 62 | const transports: any[] = [new winston.transports.Console(consoleTransport)]; 63 | 64 | if (config.nodeEnv === "test") { 65 | consoleTransport.silent = true; 66 | } else { 67 | transports.push( 68 | new winstonDailyFile({ 69 | filename: `%DATE%.log`, 70 | datePattern: "YYYY-MM-DD-HH", 71 | level: "debug", 72 | dirname: "logs", 73 | handleExceptions: true, 74 | }), 75 | ); 76 | } 77 | 78 | export const WINSTON_LOGGER_OPTIONS: LoggerOptions = { 79 | transports, 80 | }; 81 | 82 | /** 83 | * DO NOT import this one directly 84 | */ 85 | export const WINSTON_LOGGER = winston.createLogger(WINSTON_LOGGER_OPTIONS); 86 | -------------------------------------------------------------------------------- /src/core/repo-auth/repo-auth.ts: -------------------------------------------------------------------------------- 1 | import basicAuth from "basic-auth"; 2 | import { Cred } from "nodegit"; 3 | 4 | import { SecureUtils } from "../../utils"; 5 | 6 | export const AUTH_HEADERS = { 7 | generic: "x-authorization", 8 | github: "x-github-token", 9 | }; 10 | 11 | /** 12 | * Class that handle repository authentication 13 | * Support generic server auth using the `x-authentication` header which is just a pass thru to the repo 14 | * Supoort a few helper 15 | * - `x-github-token`: Pass a github token to authenticate 16 | */ 17 | export class RepoAuth { 18 | private readonly username?: string; 19 | private readonly password?: string; 20 | 21 | constructor(obj?: { username: string; password: string }) { 22 | if (obj) { 23 | this.username = obj.username; 24 | this.password = obj.password; 25 | } 26 | } 27 | 28 | /** 29 | * Get a repo auth instance from the header 30 | * @param headers: Header string map 31 | * 32 | * @returns {RepoAuth} if it manage to create an object 33 | * @returns {undefined} if the authorization headers are invalid. This should result in an error. This is not the same as not providing any headers. This means the headers were invalid 34 | */ 35 | public static fromHeaders(headers: { [key: string]: string }): RepoAuth | undefined { 36 | if (headers[AUTH_HEADERS.generic]) { 37 | const auth = parseAuthorizationHeader(headers[AUTH_HEADERS.generic]); 38 | if (!auth) { 39 | return undefined; 40 | } 41 | return new RepoAuth(auth); 42 | } else if (headers[AUTH_HEADERS.github]) { 43 | return new RepoAuth({ 44 | username: headers[AUTH_HEADERS.github], 45 | password: "x-oauth-basic", 46 | }); 47 | } 48 | return new RepoAuth(); 49 | } 50 | 51 | public toCreds(): Cred { 52 | if (this.username && this.password) { 53 | return Cred.userpassPlaintextNew(this.username, this.password); 54 | } 55 | return Cred.defaultNew(); 56 | } 57 | 58 | public toAuthorizationHeader(): string | undefined { 59 | if (this.username && this.password) { 60 | const header = `${this.username}:${this.password}`; 61 | return `Basic ${Buffer.from(header).toString("base64")}`; 62 | } 63 | return undefined; 64 | } 65 | 66 | public hash(): string | undefined { 67 | const header = this.toAuthorizationHeader(); 68 | 69 | return header ? SecureUtils.sha512(header) : undefined; 70 | } 71 | } 72 | 73 | /** 74 | * Parse the authorization header into username and password. 75 | * Needs to be in this format for now 76 | * `Basic [base64Encoded(username:password)` 77 | */ 78 | function parseAuthorizationHeader(header: string) { 79 | const result = basicAuth.parse(header); 80 | if (!result) { 81 | return undefined; 82 | } 83 | return { username: result.name, password: result.pass }; 84 | } 85 | -------------------------------------------------------------------------------- /src/services/repo-index/repo-index.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from "@nestjs/common"; 2 | import { Connection, Repository } from "typeorm"; 3 | 4 | import { Logger } from "../../core"; 5 | import { RepoReferenceRecord } from "../../models"; 6 | 7 | export interface RepoReference { 8 | readonly path: string; 9 | readonly lastUse: number; 10 | readonly lastFetch?: number; 11 | } 12 | 13 | const FETCH_CACHE_EXPIRY = 30_000; // 30s; 14 | 15 | /** 16 | * Service that keep an index on all repository cached. It presisted across restart of the server. 17 | * It keeps track of a few details related to the repository such as the last time 18 | * it was open/fetched to find least recently used repos or if a fetch is due 19 | */ 20 | @Injectable() 21 | export class RepoIndexService { 22 | private logger = new Logger(RepoIndexService); 23 | private readonly repos = new Map(); 24 | private repository: Repository; 25 | 26 | constructor(connection: Connection) { 27 | this.repository = connection.getRepository(RepoReferenceRecord); 28 | this.init().catch(e => { 29 | this.logger.error("Failed to load data from database", e); 30 | }); 31 | } 32 | 33 | public get size() { 34 | return this.repos.size; 35 | } 36 | 37 | private async update(ref: RepoReference) { 38 | this.repos.set(ref.path, ref); 39 | 40 | try { 41 | await this.repository.insert(ref); 42 | } catch (e) { 43 | await this.repository.update({ path: ref.path }, ref); 44 | } 45 | } 46 | 47 | public getLeastUsedRepos(count = 1): string[] { 48 | return [...this.repos.values()] 49 | .sort((a, b) => a.lastUse - b.lastUse) 50 | .slice(0, count) 51 | .map(x => x.path); 52 | } 53 | 54 | public needToFetch(repoId: string): boolean { 55 | const repo = this.repos.get(repoId); 56 | if (!repo || !repo.lastFetch) { 57 | return true; 58 | } 59 | const now = Date.now(); 60 | return now - repo.lastFetch > FETCH_CACHE_EXPIRY; 61 | } 62 | 63 | public markRepoAsOpened(repoId: string) { 64 | const now = Date.now(); 65 | const existing = this.repos.get(repoId); 66 | this.tryAndLog(() => this.update({ ...existing, path: repoId, lastUse: now })); 67 | } 68 | 69 | public markRepoAsFetched(repoId: string) { 70 | const now = Date.now(); 71 | const existing = this.repos.get(repoId); 72 | this.tryAndLog(() => this.update({ ...existing, path: repoId, lastFetch: now, lastUse: now })); 73 | } 74 | 75 | public markRepoAsRemoved(repoId: string) { 76 | this.repos.delete(repoId); 77 | this.tryAndLog(() => this.repository.delete({ path: repoId })); 78 | } 79 | 80 | private async init() { 81 | const repos = await this.repository.find(); 82 | for (const repo of repos) { 83 | this.repos.set(repo.path, { ...repo }); 84 | } 85 | } 86 | 87 | private tryAndLog(f: () => Promise) { 88 | f().catch(error => { 89 | this.logger.error("Error occured", error); 90 | }); 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/middlewares/logger-interceptor.ts: -------------------------------------------------------------------------------- 1 | import { CallHandler, ExecutionContext, HttpException, HttpStatus, Injectable, NestInterceptor } from "@nestjs/common"; 2 | import { Request } from "express"; 3 | import { Observable, throwError } from "rxjs"; 4 | import { catchError, tap } from "rxjs/operators"; 5 | 6 | import { Configuration } from "../config"; 7 | import { LogMetadata, Logger, Telemetry } from "../core"; 8 | 9 | @Injectable() 10 | export class LoggingInterceptor implements NestInterceptor { 11 | private logger = new Logger("Request"); 12 | 13 | constructor(private config: Configuration, private telemetry: Telemetry) {} 14 | 15 | public intercept(context: ExecutionContext, next: CallHandler): Observable { 16 | const now = Date.now(); 17 | const req: Request = context.switchToHttp().getRequest(); 18 | 19 | const commonProperties = { 20 | path: req.route.path, 21 | method: req.method, 22 | }; 23 | 24 | return next.handle().pipe( 25 | tap(() => { 26 | const response = context.switchToHttp().getResponse(); 27 | const duration = Date.now() - now; 28 | 29 | const properties = { 30 | ...commonProperties, 31 | duration, 32 | status: response.statusCode, 33 | }; 34 | 35 | const message = `${req.method} ${response.statusCode} ${req.originalUrl} (${duration}ms)`; 36 | this.logger.info(message, this.clean({ ...properties, uri: req.originalUrl })); 37 | this.trackRequest(properties); 38 | }), 39 | catchError((error: Error | HttpException) => { 40 | const statusCode = error instanceof HttpException ? error.getStatus() : HttpStatus.INTERNAL_SERVER_ERROR; 41 | const duration = Date.now() - now; 42 | const message = `${req.method} ${statusCode} ${req.originalUrl} (${duration}ms)`; 43 | 44 | const properties = { 45 | ...commonProperties, 46 | duration, 47 | status: statusCode, 48 | }; 49 | 50 | this.trackRequest(properties); 51 | if (statusCode >= 500) { 52 | this.logger.error(message, this.clean({ ...properties, uri: req.originalUrl })); 53 | this.telemetry.emitMetric({ 54 | name: "EXCEPTIONS", 55 | value: 1, 56 | dimensions: { 57 | type: error.name, 58 | path: properties.path, 59 | method: properties.method, 60 | }, 61 | }); 62 | } else { 63 | this.logger.info(message, this.clean(properties)); 64 | } 65 | return throwError(error); 66 | }), 67 | ); 68 | } 69 | 70 | private trackRequest(properties: { duration: number; path: string; method: string; status: number }) { 71 | const { duration, status, ...otherProperties } = properties; 72 | 73 | this.telemetry.emitMetric({ 74 | name: "INCOMING_REQUEST", 75 | value: duration, 76 | dimensions: { 77 | status: status.toString(), 78 | ...otherProperties, 79 | }, 80 | }); 81 | } 82 | 83 | private clean(meta: LogMetadata) { 84 | return this.config.nodeEnv === "development" ? {} : meta; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | /* Basic Options */ 4 | "target": "es2019" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */, 5 | "module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */, 6 | // "lib": [], /* Specify library files to be included in the compilation. */ 7 | // "allowJs": true, /* Allow javascript files to be compiled. */ 8 | // "checkJs": true, /* Report errors in .js files. */ 9 | // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ 10 | // "declaration": true, /* Generates corresponding '.d.ts' file. */ 11 | // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ 12 | // "sourceMap": true, /* Generates corresponding '.map' file. */ 13 | // "outFile": "./", /* Concatenate and emit output to single file. */ 14 | // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ 15 | // "composite": true, /* Enable project compilation */ 16 | // "incremental": true, /* Enable incremental compilation */ 17 | // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ 18 | // "removeComments": true, /* Do not emit comments to output. */ 19 | // "noEmit": true, /* Do not emit outputs. */ 20 | // "importHelpers": true, /* Import emit helpers from 'tslib'. */ 21 | // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ 22 | // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ 23 | 24 | /* Strict Type-Checking Options */ 25 | "strict": true /* Enable all strict type-checking options. */, 26 | 27 | "incremental": true, 28 | "tsBuildInfoFile": "./buildcache/backend.tsbuildinfo", 29 | "skipDefaultLibCheck": true, 30 | 31 | /* Additional Checks */ 32 | "noUnusedLocals": true, 33 | "noUnusedParameters": true, 34 | "noImplicitReturns": true, 35 | "skipLibCheck": true, 36 | 37 | "allowSyntheticDefaultImports": true /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */, 38 | "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */, 39 | 40 | /* Source Map Options */ 41 | "sourceMap": true, 42 | 43 | /* Experimental Options */ 44 | "experimentalDecorators": true /* Enables experimental support for ES7 decorators. */, 45 | "emitDecoratorMetadata": true /* Enables experimental support for emitting type metadata for decorators. */ 46 | }, 47 | "files": ["src/definitions.d.ts"], 48 | "include": ["src/**/*.ts", "test/**/*.ts"], 49 | "exclude": ["node_modules", "tmp", "bin"] 50 | } 51 | -------------------------------------------------------------------------------- /src/services/repo-cleanup/repo-cleanup.service.test.ts: -------------------------------------------------------------------------------- 1 | import { BehaviorSubject, Subscription } from "rxjs"; 2 | 3 | import { DiskUsage } from "../disk-usage"; 4 | import { RepoCleanupService } from "./repo-cleanup.service"; 5 | 6 | const defaultDiskUsage: DiskUsage = { 7 | total: 10_000, 8 | available: 8_000, 9 | used: 2_000, 10 | }; 11 | 12 | describe("RepoCleanupService", () => { 13 | let service: RepoCleanupService; 14 | 15 | const indexServiceSpy = { 16 | size: 20, 17 | getLeastUsedRepos: jest.fn(() => ["foo-1", "foo-2"]), 18 | }; 19 | 20 | const repoServiceSpy = { 21 | deleteLocalRepo: jest.fn(), 22 | }; 23 | 24 | const diskUsageService = { 25 | dataDiskUsage: new BehaviorSubject(defaultDiskUsage), 26 | }; 27 | 28 | let sub: Subscription; 29 | 30 | beforeEach(() => { 31 | jest.clearAllMocks(); 32 | jest.useFakeTimers(); 33 | diskUsageService.dataDiskUsage.next(defaultDiskUsage); 34 | service = new RepoCleanupService(indexServiceSpy as any, repoServiceSpy as any, diskUsageService as any); 35 | sub = service.start(); 36 | }); 37 | 38 | afterEach(() => { 39 | sub.unsubscribe(); 40 | jest.useRealTimers(); 41 | }); 42 | 43 | it("Shouldn't do anything if there is enough disk available", () => { 44 | expect(indexServiceSpy.getLeastUsedRepos).not.toHaveBeenCalled(); 45 | diskUsageService.dataDiskUsage.next({ 46 | total: 10_000, 47 | available: 5_000, 48 | used: 5_000, 49 | }); 50 | expect(indexServiceSpy.getLeastUsedRepos).not.toHaveBeenCalled(); 51 | diskUsageService.dataDiskUsage.next({ 52 | total: 10_000, 53 | available: 8_000, 54 | used: 2_000, 55 | }); 56 | expect(indexServiceSpy.getLeastUsedRepos).not.toHaveBeenCalled(); 57 | }); 58 | 59 | it("Should try to delete repos if there is not enough space", () => { 60 | expect(indexServiceSpy.getLeastUsedRepos).not.toHaveBeenCalled(); 61 | diskUsageService.dataDiskUsage.next({ 62 | total: 10_000, 63 | available: 500, 64 | used: 9_500, 65 | }); 66 | expect(indexServiceSpy.getLeastUsedRepos).toHaveBeenCalledTimes(1); 67 | expect(indexServiceSpy.getLeastUsedRepos).toHaveBeenCalledWith(1); 68 | 69 | jest.runAllTicks(); 70 | expect(repoServiceSpy.deleteLocalRepo).toHaveBeenCalledTimes(2); 71 | expect(repoServiceSpy.deleteLocalRepo).toHaveBeenCalledWith("foo-1"); 72 | expect(repoServiceSpy.deleteLocalRepo).toHaveBeenCalledWith("foo-2"); 73 | }); 74 | 75 | it("Should wait for the bathc of deletion to complete before checking the disk usage again", () => { 76 | expect(indexServiceSpy.getLeastUsedRepos).not.toHaveBeenCalled(); 77 | diskUsageService.dataDiskUsage.next({ 78 | total: 10_000, 79 | available: 500, 80 | used: 9_500, 81 | }); 82 | expect(indexServiceSpy.getLeastUsedRepos).toHaveBeenCalledTimes(1); 83 | 84 | diskUsageService.dataDiskUsage.next({ 85 | total: 10_000, 86 | available: 400, 87 | used: 9_600, 88 | }); 89 | 90 | expect(indexServiceSpy.getLeastUsedRepos).toHaveBeenCalledTimes(1); 91 | diskUsageService.dataDiskUsage.next({ 92 | total: 10_000, 93 | available: 300, 94 | used: 9_700, 95 | }); 96 | 97 | expect(indexServiceSpy.getLeastUsedRepos).toHaveBeenCalledTimes(1); 98 | 99 | jest.runAllTicks(); 100 | expect(repoServiceSpy.deleteLocalRepo).toHaveBeenCalledTimes(2); 101 | expect(repoServiceSpy.deleteLocalRepo).toHaveBeenCalledWith("foo-1"); 102 | expect(repoServiceSpy.deleteLocalRepo).toHaveBeenCalledWith("foo-2"); 103 | }); 104 | }); 105 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "git-rest-api", 3 | "version": "0.3.5", 4 | "description": "This project welcomes contributions and suggestions. Most contributions require you to agree to a\r Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us\r the rights to use your contribution. For details, visit https://cla.microsoft.com.", 5 | "main": "bin/main.js", 6 | "scripts": { 7 | "get-version": "echo $npm_package_version", 8 | "build": "tsc -p config/tsconfig.build.json", 9 | "build:watch": "npm run -s build -- --watch", 10 | "start": "npm run build && node ./bin/main.js", 11 | "start:prod": "node ./bin/main.js", 12 | "start:dev": "concurrently --handle-input \"wait-on bin/main.js && nodemon\" \"tsc -w -p tsconfig.build.json\" ", 13 | "start:debug": "nodemon --config nodemon-debug.json", 14 | "test": "jest", 15 | "test:ci": "jest --ci --reporters=default --reporters=jest-junit", 16 | "lint": "tslint -p tsconfig.json", 17 | "test:watch": "jest --watch --coverage=false --config ./config/jest.dev.config.js", 18 | "test:e2e": "jest --config ./config/jest.e2e.config.js --runInBand", 19 | "swagger:gen": "node ./bin/scripts/generate-swagger-specs.js", 20 | "autorest": "cross-var autorest sdk/config.yaml --package-version=$npm_package_version", 21 | "sdk:gen": "npm run sdk:gen:ts", 22 | "sdk:gen:ts": "rimraf ./sdk/out/typescript/* && copyfiles -u 3 \"./sdk/src/typescript/**/*\" ./sdk/out/typescript && npm run autorest -- --typescript && cross-var npm version --prefix ./sdk/out/typescript $npm_package_version && npm run prepack --prefix ./sdk/out/typescript" 23 | }, 24 | "repository": { 25 | "type": "git", 26 | "url": "git+https://github.com/Azure/git-rest-api.git" 27 | }, 28 | "author": "", 29 | "license": "MIT", 30 | "bugs": { 31 | "url": "https://github.com/Azure/git-rest-api/issues" 32 | }, 33 | "homepage": "https://github.com/Azure/git-rest-api#readme", 34 | "devDependencies": { 35 | "@types/basic-auth": "^1.1.2", 36 | "@types/cls-hooked": "^4.3.0", 37 | "@types/convict": "^4.2.1", 38 | "@types/helmet": "0.0.43", 39 | "@types/jest": "^24.0.13", 40 | "@types/node-fetch": "^2.3.5", 41 | "@types/nodegit": "^0.26.1", 42 | "@types/rimraf": "^2.0.2", 43 | "@types/triple-beam": "^1.3.0", 44 | "@types/uuid": "^3.4.4", 45 | "autorest": "^2.0.4283", 46 | "concurrently": "^6.2.1", 47 | "copyfiles": "^2.4.1", 48 | "cross-var": "^1.1.0", 49 | "jest": "^27.4.4", 50 | "jest-junit": "^6.4.0", 51 | "prettier": "^1.18.2", 52 | "ts-jest": "^27.1.1", 53 | "tslint": "^5.17.0", 54 | "tslint-config-prettier": "^1.18.0", 55 | "tslint-plugin-prettier": "^2.0.1", 56 | "typescript": "^4.5.3" 57 | }, 58 | "dependencies": { 59 | "@nestjs/common": "^8.0.7", 60 | "@nestjs/core": "^8.0.7", 61 | "@nestjs/platform-express": "^8.4.7", 62 | "@nestjs/swagger": "^5.0.9", 63 | "@types/node": "^12.0.8", 64 | "basic-auth": "^2.0.1", 65 | "chalk": "^2.4.2", 66 | "class-validator": "^0.13.2", 67 | "cls-hooked": "^4.2.2", 68 | "convict": "^6.2.3", 69 | "diskusage": "^1.1.2", 70 | "express": "^4.17.1", 71 | "fast-safe-stringify": "^2.0.6", 72 | "helmet": "^5.0.2", 73 | "hot-shots": "^6.3.0", 74 | "node-fetch": "^2.6.7", 75 | "nodegit": "^0.26.4", 76 | "reflect-metadata": "^0.1.13", 77 | "rimraf": "^2.6.3", 78 | "rxjs": "^7.4.0", 79 | "sqlite3": "^5.0.3", 80 | "swagger-ui-express": "^4.3.0", 81 | "triple-beam": "^1.3.0", 82 | "typeorm": "^ 0.3.0", 83 | "uuid": "^3.3.2", 84 | "winston": "^3.2.1", 85 | "winston-daily-rotate-file": "^3.9.0" 86 | }, 87 | "jest-junit": { 88 | "outputDirectory": ".", 89 | "outputName": "test-results.xml" 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /test/e2e/custom-matchers.ts: -------------------------------------------------------------------------------- 1 | // tslint:disable: no-implicit-dependencies 2 | import { MatcherState } from "expect"; 3 | import fs from "fs"; 4 | import SnapshotState from "jest-snapshot/build/State"; 5 | import path from "path"; 6 | 7 | declare global { 8 | namespace jest { 9 | interface Matchers { 10 | toMatchSpecificSnapshot(snapshotFile: string): CustomMatcherResult; 11 | toMatchPayload(payloadName: string): CustomMatcherResult; 12 | } 13 | } 14 | } 15 | 16 | function getAbsolutePathToSnapshot(testPath: string, snapshotFile: string) { 17 | return path.isAbsolute(snapshotFile) ? snapshotFile : path.resolve(path.dirname(testPath), snapshotFile); 18 | } 19 | 20 | function serializeContent(content: object | object[]) { 21 | return JSON.stringify(content, null, 2); 22 | } 23 | 24 | type Context = jest.MatcherUtils & 25 | MatcherState & { 26 | snapshotState: SnapshotState; 27 | }; 28 | 29 | /** 30 | * Helper 31 | */ 32 | function toMatchPayload(this: Context, received: object | object[], payload: string): jest.CustomMatcherResult { 33 | return toMatchSpecificSnapshot.call(this, received, `__snapshots__/api/${payload}.json`); 34 | } 35 | 36 | function toMatchSpecificSnapshot( 37 | this: Context, 38 | received: object | object[], 39 | filename: string, 40 | ): jest.CustomMatcherResult { 41 | if (this.isNot) { 42 | return { 43 | pass: true, // Will get inverted because of the .not 44 | message: () => `.${this.utils.BOLD_WEIGHT("not")} cannot be used with snapshot matchers`, 45 | }; 46 | } 47 | 48 | const filepath = getAbsolutePathToSnapshot(this.testPath!, filename); 49 | const content = serializeContent(received); 50 | const updateSnapshot: "none" | "all" | "new" = (this.snapshotState as any)._updateSnapshot; 51 | 52 | const coloredFilename = this.utils.DIM_COLOR(filename); 53 | const errorColor = this.utils.RECEIVED_COLOR; 54 | 55 | if (updateSnapshot === "none" && !fs.existsSync(filepath)) { 56 | // We're probably running in CI environment 57 | 58 | this.snapshotState.unmatched++; 59 | 60 | return { 61 | pass: false, 62 | message: () => 63 | `New output file ${coloredFilename} was ${errorColor("not written")}.\n\n` + 64 | "The update flag must be explicitly passed to write a new snapshot.\n\n", 65 | }; 66 | } 67 | 68 | if (fs.existsSync(filepath)) { 69 | const output = fs.readFileSync(filepath, "utf8").replace(/\r\n/g, "\n"); 70 | // The matcher is being used with `.not` 71 | if (output === content) { 72 | this.snapshotState.matched++; 73 | return { pass: true, message: () => "" }; 74 | } else { 75 | if (updateSnapshot === "all") { 76 | fs.mkdirSync(path.dirname(filepath), { recursive: true }); 77 | fs.writeFileSync(filepath, content); 78 | 79 | this.snapshotState.updated++; 80 | 81 | return { pass: true, message: () => "" }; 82 | } else { 83 | this.snapshotState.unmatched++; 84 | 85 | return { 86 | pass: false, 87 | message: () => 88 | `Received content ${errorColor("doesn't match")} the file ${coloredFilename}.\n\n${this.utils.diff( 89 | content, 90 | output, 91 | )}`, 92 | }; 93 | } 94 | } 95 | } else { 96 | if (updateSnapshot === "new" || updateSnapshot === "all") { 97 | fs.mkdirSync(path.dirname(filepath), { recursive: true }); 98 | fs.writeFileSync(filepath, content); 99 | 100 | this.snapshotState.added++; 101 | 102 | return { pass: true, message: () => "" }; 103 | } else { 104 | this.snapshotState.unmatched++; 105 | 106 | return { 107 | pass: true, 108 | message: () => `The output file ${coloredFilename} ${errorColor("doesn't exist")}.`, 109 | }; 110 | } 111 | } 112 | } 113 | 114 | expect.extend({ toMatchSpecificSnapshot, toMatchPayload } as any); 115 | -------------------------------------------------------------------------------- /src/services/permission/permission.service.test.ts: -------------------------------------------------------------------------------- 1 | import { RepoAuth } from "../../core"; 2 | import { PermissionCacheService } from "./cache"; 3 | import { PermissionService } from "./permission.service"; 4 | import { GitRemotePermission } from "./permissions"; 5 | 6 | const remote = "github.com/Azure/some-repo"; 7 | const remotePrivate = "github.com/Azure/some-private"; 8 | const auth = new RepoAuth({ username: "token-1", password: "x-oauth-token" }); 9 | 10 | describe("PermissionService", () => { 11 | let service: PermissionService; 12 | let cache: PermissionCacheService; 13 | const httpSpy = { 14 | fetch: jest.fn(uri => { 15 | if (uri.includes(remote) && uri.includes("git-upload-pack")) { 16 | return { status: 200 }; 17 | } else if (uri.includes(remotePrivate)) { 18 | return { status: 200 }; 19 | } 20 | return { 21 | status: 404, 22 | }; 23 | }), 24 | }; 25 | 26 | beforeEach(() => { 27 | jest.clearAllMocks(); 28 | cache = new PermissionCacheService(); 29 | service = new PermissionService(cache, httpSpy as any); 30 | }); 31 | 32 | it("returns the permission in the cache if its there", async () => { 33 | cache.set(auth, remote, GitRemotePermission.Read); 34 | expect(await service.get(auth, remote)).toBe(GitRemotePermission.Read); 35 | }); 36 | 37 | it("returns none when permission is set to none", async () => { 38 | service.set(auth, remote, GitRemotePermission.None); 39 | expect(await service.get(auth, remote)).toBe(GitRemotePermission.None); 40 | }); 41 | 42 | describe("when permission are not cached", () => { 43 | it("try write endpoint", async () => { 44 | const permission = await service.get(auth, remotePrivate); 45 | expect(permission).toBe(GitRemotePermission.Write); 46 | expect(cache.get(auth, remotePrivate)).toBe(GitRemotePermission.Write); 47 | 48 | expect(httpSpy.fetch).toHaveBeenCalledTimes(1); 49 | 50 | expect(httpSpy.fetch).toHaveBeenCalledWith("https://github.com/Azure/some-private.git/git-receive-pack", { 51 | headers: { 52 | Authorization: auth.toAuthorizationHeader(), 53 | }, 54 | method: "POST", 55 | }); 56 | }); 57 | 58 | it("try write endpoint then read if can't write", async () => { 59 | const permission = await service.get(auth, remote); 60 | expect(permission).toBe(GitRemotePermission.Read); 61 | expect(cache.get(auth, remote)).toBe(GitRemotePermission.Read); 62 | 63 | expect(httpSpy.fetch).toHaveBeenCalledTimes(2); 64 | 65 | expect(httpSpy.fetch).toHaveBeenCalledWith("https://github.com/Azure/some-repo.git/git-receive-pack", { 66 | headers: { 67 | Authorization: auth.toAuthorizationHeader(), 68 | }, 69 | method: "POST", 70 | }); 71 | 72 | expect(httpSpy.fetch).toHaveBeenCalledWith("https://github.com/Azure/some-repo.git/git-upload-pack", { 73 | headers: { 74 | Authorization: auth.toAuthorizationHeader(), 75 | }, 76 | method: "POST", 77 | }); 78 | }); 79 | 80 | it("try write and read if has no permission", async () => { 81 | const permission = await service.get(auth, "github.com/Azure/other-repo-no-permissions"); 82 | expect(permission).toBe(GitRemotePermission.None); 83 | expect(cache.get(auth, "github.com/Azure/other-repo-no-permissions")).toBe(GitRemotePermission.None); 84 | 85 | expect(httpSpy.fetch).toHaveBeenCalledTimes(2); 86 | 87 | expect(httpSpy.fetch).toHaveBeenCalledWith( 88 | "https://github.com/Azure/other-repo-no-permissions.git/git-receive-pack", 89 | { 90 | headers: { 91 | Authorization: auth.toAuthorizationHeader(), 92 | }, 93 | method: "POST", 94 | }, 95 | ); 96 | 97 | expect(httpSpy.fetch).toHaveBeenCalledWith( 98 | "https://github.com/Azure/other-repo-no-permissions.git/git-upload-pack", 99 | { 100 | headers: { 101 | Authorization: auth.toAuthorizationHeader(), 102 | }, 103 | method: "POST", 104 | }, 105 | ); 106 | }); 107 | }); 108 | }); 109 | -------------------------------------------------------------------------------- /src/services/repo/repo.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable, NotFoundException } from "@nestjs/common"; 2 | import { Repository } from "nodegit"; 3 | import path from "path"; 4 | 5 | import { Configuration } from "../../config"; 6 | import { Logger, RepoAuth } from "../../core"; 7 | import { FSService } from "../fs"; 8 | import { GitRemotePermission, PermissionService } from "../permission"; 9 | import { RepoIndexService } from "../repo-index"; 10 | import { LocalRepo, RemoteDef } from "./local-repo/local-repo"; 11 | 12 | export interface GitBaseOptions { 13 | auth?: RepoAuth; 14 | } 15 | 16 | @Injectable() 17 | export class RepoService { 18 | public readonly repoCacheFolder = path.join(this.config.dataDir, "repos"); 19 | 20 | /** 21 | * Map that contains a key and promise when cloning a given repo 22 | */ 23 | private openedRepos = new Map(); 24 | private deletingRepos = new Map>(); 25 | 26 | private logger = new Logger(RepoService); 27 | 28 | constructor( 29 | private config: Configuration, 30 | private fs: FSService, 31 | private repoIndexService: RepoIndexService, 32 | private permissionService: PermissionService, 33 | ) {} 34 | 35 | public async use(remote: string, options: GitBaseOptions, action: (repo: Repository) => Promise): Promise { 36 | await this.validatePermissions([remote], options); 37 | const repoPath = this.getRepoMainPath(remote); 38 | const origin = { 39 | name: "origin", 40 | remote, 41 | }; 42 | return this.useWithRemotes(repoPath, options, [origin], action); 43 | } 44 | 45 | public async useForCompare( 46 | base: RemoteDef, 47 | head: RemoteDef, 48 | options: GitBaseOptions = {}, 49 | action: (repo: Repository) => Promise, 50 | ): Promise { 51 | await this.validatePermissions([base.remote, head.remote], options); 52 | const localName = `${base.remote}-${head.remote}`; 53 | const repoPath = this.getRepoMainPath(localName, "compare"); 54 | 55 | return this.useWithRemotes(repoPath, options, [base, head], action); 56 | } 57 | 58 | public async deleteLocalRepo(repoPath: string): Promise { 59 | const existingDeletion = this.deletingRepos.get(repoPath); 60 | if (existingDeletion) { 61 | return existingDeletion; 62 | } 63 | if (this.openedRepos.has(repoPath)) { 64 | this.logger.info("Can't delete this repo has its opened"); 65 | return false; 66 | } 67 | 68 | const promise = this.fs 69 | .rm(repoPath) 70 | .then(() => true) 71 | .finally(() => { 72 | this.deletingRepos.delete(repoPath); 73 | this.repoIndexService.markRepoAsRemoved(repoPath); 74 | }); 75 | 76 | this.deletingRepos.set(repoPath, promise); 77 | return promise; 78 | } 79 | 80 | private async useWithRemotes( 81 | repoPath: string, 82 | options: GitBaseOptions, 83 | remotes: RemoteDef[], 84 | action: (repo: Repository) => Promise, 85 | ): Promise { 86 | this.repoIndexService.markRepoAsOpened(repoPath); 87 | let repo = this.openedRepos.get(repoPath); 88 | 89 | // If repo is deleting wait for it to be deleted and reinit again 90 | const deletion = this.deletingRepos.get(repoPath); 91 | if (deletion) { 92 | await deletion; 93 | } 94 | 95 | if (!repo) { 96 | repo = new LocalRepo(repoPath, this.fs, this.repoIndexService); 97 | this.openedRepos.set(repoPath, repo); 98 | repo.onDestroy.subscribe(() => { 99 | this.openedRepos.delete(repoPath); 100 | }); 101 | await repo.init(remotes); 102 | } else { 103 | repo.ref(); 104 | } 105 | if (this.repoIndexService.needToFetch(repoPath)) { 106 | await repo.update(options); 107 | } 108 | const response = await repo.use(action); 109 | repo.unref(); 110 | return response; 111 | } 112 | 113 | public async validatePermissions(remotes: string[], options: GitBaseOptions) { 114 | await Promise.all( 115 | remotes.map(async remote => { 116 | const permission = await this.permissionService.get(options.auth || new RepoAuth(), remote); 117 | if (permission === GitRemotePermission.None) { 118 | throw new NotFoundException(`Cannot find or missing permission to access '${remote}'`); 119 | } 120 | }), 121 | ); 122 | } 123 | 124 | private getRepoMainPath(remote: string, namespace?: string) { 125 | if (namespace) { 126 | return path.join(this.repoCacheFolder, namespace, encodeURIComponent(remote)); 127 | } 128 | return path.join(this.repoCacheFolder, encodeURIComponent(remote)); 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | | CI | CD | 2 | | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | 3 | | [![Build Status](https://dev.azure.com/azure-sdk/public/_apis/build/status/tools/public.git-rest-api.ci?branchName=master)](https://dev.azure.com/azure-sdk/public/_build/latest?definitionId=444&branchName=master) | [![Build Status](https://dev.azure.com/azure-sdk/public/_apis/build/status/tools/Azure.git-rest-api%20Build?branchName=master)](https://dev.azure.com/azure-sdk/public/_build/latest?definitionId=445&branchName=master) | 4 | 5 | - Javascript SDK ![https://www.npmjs.com/package/git-rest-api-sdk](https://img.shields.io/npm/v/git-rest-api-sdk.svg) 6 | 7 | # Overview 8 | 9 | This project is webserver that can be used as a docker container to provide a proxy/rest api on top of git repository. 10 | One of the goal of this is to work around github api rate limiting. The api tries to looks as similar to the github rest api. 11 | 12 | Following apis are supported for now: 13 | 14 | - Commits 15 | - List 16 | - Get 17 | - Compare 18 | - Branch 19 | - List 20 | - Files 21 | - List 22 | - Get content 23 | 24 | # Quick start 25 | 26 | ``` 27 | docker run -d -p 3009:3009 azuredevx/git-rest-api 28 | sleep 3 # optional: wait for container to start 29 | curl localhost:3009/repos/github.com%2Foctocat%2FHello-World/contents/ 30 | ``` 31 | 32 | # Deploy with docker 33 | 34 | Image: `azuredevx/git-rest-api` 35 | 36 | You can configure various options using environemnt variables. See https://github.com/Azure/git-rest-api/blob/master/src/config/schema.ts 37 | 38 | **Note: There is no authentication layer yet from your service to the docker image. You should most likely use in in a contained environment(Inside a VNet or kubernetes cluster for example)** 39 | 40 | # Use the API 41 | 42 | * Using the Javascript sdk 43 | A javascript sdk is always included and up to date with the latest version of the service. 44 | 45 | ``` 46 | npm install --save git-rest-api-sdk 47 | ``` 48 | 49 | * Use rest api 50 | 51 | There is a `/swagger` endpoint which serve the swagger UI with all the api you can use and help you figure out the available/required params. 52 | 53 | To authenticate against the repo if its not public you have 2 options: 54 | - `x-authorization`: This needs to be in a Basic auth format(`Basic base64(usename:password)`). Check with the git server how you can authenticate. 55 | - `x-github-token`: This is an helper for authnetication against github api. The basic header will be generated automatically 56 | 57 | # Develop 58 | 59 | 1. Install dependencies 60 | 61 | ```bash 62 | npm install 63 | ``` 64 | 65 | 2. Run 66 | 67 | ```bash 68 | npm start # To run once 69 | npm start:watch # To run and restart when there is a change 70 | ``` 71 | 72 | Run in vscode 73 | 74 | Instead of `npm start` run `npm build:watch` and in vscode press `F5` 75 | 76 | # Windows 77 | 78 | - Long path issue. libgit2(library behind nodegit) doesn't support windows long path feature. Which means some repo with long reference might not work on windows. You can test with other simpler repos on windows. You should however not use this in production on windows. 79 | 80 | # Contributing 81 | 82 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 83 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 84 | the rights to use your contribution. For details, visit https://cla.microsoft.com. 85 | 86 | When you submit a pull request, a CLA-bot will automatically determine whether you need to provide 87 | a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions 88 | provided by the bot. You will only need to do this once across all repos using our CLA. 89 | 90 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 91 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 92 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 93 | -------------------------------------------------------------------------------- /src/services/content/content.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable, NotFoundException } from "@nestjs/common"; 2 | import { Repository, Tree, TreeEntry } from "nodegit"; 3 | 4 | import { 5 | GitContents, 6 | GitDirObjectContent, 7 | GitFileObjectWithContent, 8 | GitFileObjectWithoutContent, 9 | GitSubmoduleObjectContent, 10 | GitTree, 11 | } from "../../dtos"; 12 | import { CommitService } from "../commit"; 13 | import { GitBaseOptions, RepoService } from "../repo"; 14 | 15 | @Injectable() 16 | export class ContentService { 17 | constructor(private repoService: RepoService, private commitService: CommitService) {} 18 | 19 | public async getContents( 20 | remote: string, 21 | path: string | undefined, 22 | ref: string | undefined = "master", 23 | recursive: boolean = false, 24 | includeContents: boolean = true, 25 | options: GitBaseOptions = {}, 26 | ): Promise { 27 | return this.repoService.use(remote, options, async repo => { 28 | return this.getGitContents(repo, path, recursive, includeContents, ref); 29 | }); 30 | } 31 | 32 | public async getGitContents( 33 | repo: Repository, 34 | path: string | undefined, 35 | recursive: boolean, 36 | includeContents: boolean, 37 | ref: string | undefined = "master", 38 | ) { 39 | const commit = await this.commitService.getCommit(repo, ref); 40 | if (!commit) { 41 | return new NotFoundException(`Ref '${ref}' not found.`); 42 | } 43 | 44 | let entries: TreeEntry[]; 45 | 46 | if (path) { 47 | try { 48 | const pathEntry = await commit.getEntry(path); 49 | 50 | if (pathEntry.isTree()) { 51 | const tree = await pathEntry.getTree(); 52 | 53 | // for directories, either 54 | if (recursive) { 55 | // recursively get children 56 | entries = await this.getAllChildEntries(tree); 57 | } else { 58 | // get children immediate children 59 | entries = tree.entries(); 60 | } 61 | } else { 62 | // for files get array of size 1 63 | entries = [await commit.getEntry(path)]; 64 | } 65 | } catch (e) { 66 | return new NotFoundException(`Path '${path}' not found.`); 67 | } 68 | } else { 69 | const tree = await commit.getTree(); 70 | entries = recursive ? await this.getAllChildEntries(tree) : tree.entries(); 71 | } 72 | 73 | return this.getEntries(entries, includeContents); 74 | } 75 | 76 | private async getFileEntryAsObject( 77 | entry: TreeEntry, 78 | includeContents: boolean, 79 | ): Promise { 80 | const blob = await entry.getBlob(); 81 | const file = { 82 | type: "file", 83 | encoding: "base64", 84 | size: blob.rawsize(), 85 | name: entry.name(), 86 | path: entry.path(), 87 | sha: entry.sha(), 88 | }; 89 | 90 | if (includeContents) { 91 | return new GitFileObjectWithContent({ 92 | ...file, 93 | content: blob.content().toString("base64"), 94 | }); 95 | } 96 | 97 | return new GitFileObjectWithoutContent(file); 98 | } 99 | 100 | private async getDirEntryAsObject(entry: TreeEntry): Promise { 101 | return new GitDirObjectContent({ 102 | type: "dir", 103 | size: 0, 104 | name: entry.name(), 105 | path: entry.path(), 106 | sha: entry.sha(), 107 | }); 108 | } 109 | private async getSubmoduleEntryAsObject(entry: TreeEntry): Promise { 110 | return new GitSubmoduleObjectContent({ 111 | type: "submodule", 112 | size: 0, 113 | name: entry.name(), 114 | path: entry.path(), 115 | sha: entry.sha(), 116 | }); 117 | } 118 | 119 | private async getEntries(entries: TreeEntry[], includeContents: boolean): Promise { 120 | const [files, dirs, submodules] = await Promise.all([ 121 | Promise.all( 122 | entries.filter(entry => entry.isFile()).map(async entry => this.getFileEntryAsObject(entry, includeContents)), 123 | ), 124 | Promise.all(entries.filter(entry => entry.isDirectory()).map(async entry => this.getDirEntryAsObject(entry))), 125 | Promise.all( 126 | entries.filter(entry => entry.isSubmodule()).map(async entry => this.getSubmoduleEntryAsObject(entry)), 127 | ), 128 | ]); 129 | 130 | if (includeContents) { 131 | return new GitContents({ files, dirs, submodules }); 132 | } 133 | 134 | return new GitTree({ files: files as GitFileObjectWithContent[], dirs, submodules }); 135 | } 136 | 137 | private async getAllChildEntries(tree: Tree): Promise { 138 | return new Promise((resolve, reject) => { 139 | const eventEmitter = tree.walk(false); 140 | 141 | eventEmitter.on("end", (trees: TreeEntry[]) => { 142 | resolve(trees); 143 | }); 144 | 145 | eventEmitter.on("error", error => { 146 | reject(error); 147 | }); 148 | 149 | eventEmitter.start(); 150 | }); 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /src/controllers/tree/__snapshots__/tree.controller.e2e.ts.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`Test content controller for path '/repos/github.com%2Ftest-repo-billy%2Fgit-api-tests/tree' 1`] = ` 4 | Object { 5 | "dirs": Array [ 6 | Object { 7 | "name": "dir1", 8 | "path": "dir1", 9 | "sha": "b638a8a4a9f44184a3a430988a9c5ef383bad364", 10 | "size": 0, 11 | "type": "dir", 12 | }, 13 | Object { 14 | "name": "dir2", 15 | "path": "dir1/dir2", 16 | "sha": "483221c9d8371862bdb2c5d452130ab5ca0534a3", 17 | "size": 0, 18 | "type": "dir", 19 | }, 20 | ], 21 | "files": Array [ 22 | Object { 23 | "encoding": "base64", 24 | "name": ".gitignore", 25 | "path": ".gitignore", 26 | "sha": "ad46b30886fa350c1f59761b100e5e4b01f9a7ec", 27 | "size": 914, 28 | "type": "file", 29 | }, 30 | Object { 31 | "encoding": "base64", 32 | "name": "LICENSE", 33 | "path": "LICENSE", 34 | "sha": "98023418cdb98210a5f71ea74ec557dbbd8f0e83", 35 | "size": 1072, 36 | "type": "file", 37 | }, 38 | Object { 39 | "encoding": "base64", 40 | "name": "README.md", 41 | "path": "README.md", 42 | "sha": "b5fd37e731f1e7931da42484ae0290554cb42c0f", 43 | "size": 78, 44 | "type": "file", 45 | }, 46 | Object { 47 | "encoding": "base64", 48 | "name": "fileA.txt", 49 | "path": "dir1/fileA.txt", 50 | "sha": "ab47708c98ac88bbdf3ca75f4730d86a84f702a2", 51 | "size": 6, 52 | "type": "file", 53 | }, 54 | Object { 55 | "encoding": "base64", 56 | "name": "fileB.txt", 57 | "path": "dir1/dir2/fileB.txt", 58 | "sha": "78ed112c991c8abeba325c039a398ba626c425ab", 59 | "size": 6, 60 | "type": "file", 61 | }, 62 | ], 63 | "submodules": Array [], 64 | } 65 | `; 66 | 67 | exports[`Test content controller for path '/repos/github.com%2Ftest-repo-billy%2Fgit-api-tests/tree/' 1`] = ` 68 | Object { 69 | "dirs": Array [ 70 | Object { 71 | "name": "dir1", 72 | "path": "dir1", 73 | "sha": "b638a8a4a9f44184a3a430988a9c5ef383bad364", 74 | "size": 0, 75 | "type": "dir", 76 | }, 77 | Object { 78 | "name": "dir2", 79 | "path": "dir1/dir2", 80 | "sha": "483221c9d8371862bdb2c5d452130ab5ca0534a3", 81 | "size": 0, 82 | "type": "dir", 83 | }, 84 | ], 85 | "files": Array [ 86 | Object { 87 | "encoding": "base64", 88 | "name": ".gitignore", 89 | "path": ".gitignore", 90 | "sha": "ad46b30886fa350c1f59761b100e5e4b01f9a7ec", 91 | "size": 914, 92 | "type": "file", 93 | }, 94 | Object { 95 | "encoding": "base64", 96 | "name": "LICENSE", 97 | "path": "LICENSE", 98 | "sha": "98023418cdb98210a5f71ea74ec557dbbd8f0e83", 99 | "size": 1072, 100 | "type": "file", 101 | }, 102 | Object { 103 | "encoding": "base64", 104 | "name": "README.md", 105 | "path": "README.md", 106 | "sha": "b5fd37e731f1e7931da42484ae0290554cb42c0f", 107 | "size": 78, 108 | "type": "file", 109 | }, 110 | Object { 111 | "encoding": "base64", 112 | "name": "fileA.txt", 113 | "path": "dir1/fileA.txt", 114 | "sha": "ab47708c98ac88bbdf3ca75f4730d86a84f702a2", 115 | "size": 6, 116 | "type": "file", 117 | }, 118 | Object { 119 | "encoding": "base64", 120 | "name": "fileB.txt", 121 | "path": "dir1/dir2/fileB.txt", 122 | "sha": "78ed112c991c8abeba325c039a398ba626c425ab", 123 | "size": 6, 124 | "type": "file", 125 | }, 126 | ], 127 | "submodules": Array [], 128 | } 129 | `; 130 | 131 | exports[`Test content controller for path '/repos/github.com%2Ftest-repo-billy%2Fgit-api-tests/tree/README.md' 1`] = ` 132 | Object { 133 | "dirs": Array [], 134 | "files": Array [ 135 | Object { 136 | "encoding": "base64", 137 | "name": "README.md", 138 | "path": "README.md", 139 | "sha": "b5fd37e731f1e7931da42484ae0290554cb42c0f", 140 | "size": 78, 141 | "type": "file", 142 | }, 143 | ], 144 | "submodules": Array [], 145 | } 146 | `; 147 | 148 | exports[`Test content controller for path '/repos/github.com%2Ftest-repo-billy%2Fgit-api-tests/tree/dir1' 1`] = ` 149 | Object { 150 | "dirs": Array [ 151 | Object { 152 | "name": "dir2", 153 | "path": "dir1/dir2", 154 | "sha": "483221c9d8371862bdb2c5d452130ab5ca0534a3", 155 | "size": 0, 156 | "type": "dir", 157 | }, 158 | ], 159 | "files": Array [ 160 | Object { 161 | "encoding": "base64", 162 | "name": "fileA.txt", 163 | "path": "dir1/fileA.txt", 164 | "sha": "ab47708c98ac88bbdf3ca75f4730d86a84f702a2", 165 | "size": 6, 166 | "type": "file", 167 | }, 168 | Object { 169 | "encoding": "base64", 170 | "name": "fileB.txt", 171 | "path": "dir1/dir2/fileB.txt", 172 | "sha": "78ed112c991c8abeba325c039a398ba626c425ab", 173 | "size": 6, 174 | "type": "file", 175 | }, 176 | ], 177 | "submodules": Array [], 178 | } 179 | `; 180 | -------------------------------------------------------------------------------- /src/services/repo/local-repo/local-repo.ts: -------------------------------------------------------------------------------- 1 | import { NotFoundException } from "@nestjs/common"; 2 | import { Cred, Fetch, FetchOptions, Remote, Repository } from "nodegit"; 3 | import { BehaviorSubject, Subject } from "rxjs"; 4 | import { debounceTime, filter } from "rxjs/operators"; 5 | 6 | import { Logger } from "../../../core"; 7 | import { FSService } from "../../fs"; 8 | import { RepoIndexService } from "../../repo-index/index"; 9 | import { StateMutex } from "../mutex"; 10 | import { GitBaseOptions } from "../repo.service"; 11 | 12 | export function credentialsCallback(options: GitBaseOptions): () => Cred { 13 | return () => { 14 | if (options.auth) { 15 | return options.auth.toCreds(); 16 | } 17 | return Cred.defaultNew(); 18 | }; 19 | } 20 | 21 | export const defaultFetchOptions: FetchOptions = { 22 | downloadTags: 0, 23 | prune: Fetch.PRUNE.GIT_FETCH_PRUNE, 24 | }; 25 | 26 | export enum LocalRepoStatus { 27 | Initializing = "initializing", 28 | Updating = "updating", 29 | Deleting = "deleting", 30 | Idle = "idle", 31 | Reading = "reading", 32 | } 33 | 34 | export interface RemoteDef { 35 | name: string; 36 | remote: string; 37 | } 38 | 39 | /** 40 | * Class referencing a local repo to manage concorrent actions and garbage collection. 41 | * **DO NOT USE outside of the `RepoService`. There must be only on instance of a LocalRepo mapping to the same repository to work correctly** 42 | * Use `RepoService` to access a local repo. 43 | */ 44 | export class LocalRepo { 45 | public onDestroy = new Subject(); 46 | 47 | private repo?: Repository; 48 | private currentUpdate?: Promise; 49 | 50 | private mutex = new StateMutex( 51 | LocalRepoStatus.Idle, 52 | LocalRepoStatus.Initializing, 53 | ); 54 | 55 | private logger = new Logger("LocalRepo"); 56 | private refs = new BehaviorSubject(1); // Automatically start with a reference 57 | 58 | constructor(public readonly path: string, private fs: FSService, private repoIndex: RepoIndexService) { 59 | this.refs 60 | .pipe( 61 | debounceTime(100), // Give a 100ms timeout before closing 62 | filter(x => x === 0), 63 | ) 64 | .subscribe(() => { 65 | this.dispose(); 66 | }); 67 | } 68 | 69 | public ref() { 70 | const refs = this.refs.value; 71 | this.refs.next(refs + 1); 72 | } 73 | 74 | public unref() { 75 | this.refs.next(this.refs.value - 1); 76 | } 77 | 78 | public dispose() { 79 | if (this.repo) { 80 | this.repo.cleanup(); 81 | } 82 | if (!this.refs.closed) { 83 | this.refs.complete(); 84 | } 85 | if (!this.onDestroy.closed) { 86 | this.onDestroy.next(null); 87 | this.onDestroy.complete(); 88 | } 89 | } 90 | 91 | public async init(remotes: RemoteDef[]): Promise { 92 | const lock = await this.mutex.lock(LocalRepoStatus.Initializing, { exclusive: true }); 93 | if (!this.repo) { 94 | this.repo = await this.loadRepo(remotes); 95 | } 96 | lock.release(); 97 | } 98 | 99 | public async update(options: GitBaseOptions = {}) { 100 | if (!this.currentUpdate) { 101 | this.currentUpdate = this.lockAndUpdate(options).then(() => { 102 | this.currentUpdate = undefined; 103 | }); 104 | } 105 | return this.currentUpdate; 106 | } 107 | 108 | public async use(action: (repo: Repository) => Promise): Promise { 109 | const lock = await this.mutex.lock(LocalRepoStatus.Reading); 110 | 111 | if (!this.repo) { 112 | throw new Error("Repo should have been loaded. Was init called"); 113 | } 114 | 115 | try { 116 | return await action(this.repo); 117 | } finally { 118 | lock.release(); 119 | } 120 | } 121 | 122 | private async loadRepo(remotes: RemoteDef[]) { 123 | if (await this.fs.exists(this.path)) { 124 | try { 125 | return await Repository.open(this.path); 126 | } catch (error) { 127 | this.logger.error("Failed to open repository. Deleting it"); 128 | await this.fs.rm(this.path); 129 | } 130 | } 131 | 132 | const repo = await Repository.init(this.path, 1); 133 | for (const { name, remote: value } of remotes) { 134 | await Remote.create(repo, name, `https://${value}`); 135 | } 136 | return repo; 137 | } 138 | 139 | private async lockAndUpdate(options: GitBaseOptions = {}) { 140 | const lock = await this.mutex.lock(LocalRepoStatus.Updating, { exclusive: true }); 141 | if (!this.repo) { 142 | throw new Error("Repo should have been loaded. Was init called"); 143 | } 144 | try { 145 | await this.updateRepo(this.repo, options); 146 | } finally { 147 | lock.release(); 148 | } 149 | } 150 | 151 | private async updateRepo(repo: Repository, options: GitBaseOptions) { 152 | try { 153 | await repo.fetchAll({ 154 | ...defaultFetchOptions, 155 | callbacks: { 156 | credentials: credentialsCallback(options), 157 | }, 158 | }); 159 | this.repoIndex.markRepoAsFetched(this.path); 160 | } catch { 161 | throw new NotFoundException(); 162 | } 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /src/services/commit/commit.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable, NotFoundException } from "@nestjs/common"; 2 | import { Commit, Oid, Repository, Revwalk, Signature, Time } from "nodegit"; 3 | 4 | import { PaginatedList, Pagination, getPage, getPaginationSkip } from "../../core"; 5 | import { GitCommit, GitCommitRef, GitSignature } from "../../dtos"; 6 | import { GitBaseOptions, RepoService } from "../repo"; 7 | 8 | const LIST_COMMIT_PAGE_SIZE = 100; 9 | 10 | export interface ListCommitsOptions { 11 | pagination?: Pagination; 12 | ref?: string; 13 | } 14 | 15 | @Injectable() 16 | export class CommitService { 17 | constructor(private repoService: RepoService) {} 18 | 19 | public async list( 20 | remote: string, 21 | options: ListCommitsOptions & GitBaseOptions = {}, 22 | ): Promise | NotFoundException> { 23 | return this.repoService.use(remote, options, async repo => { 24 | const commits = await this.listCommits(repo, options); 25 | if (commits instanceof NotFoundException) { 26 | return commits; 27 | } 28 | 29 | const items = await Promise.all(commits.items.map(async x => toGitCommit(x))); 30 | return { 31 | ...commits, 32 | items, 33 | }; 34 | }); 35 | } 36 | 37 | public async get(remote: string, commitSha: string, options: GitBaseOptions = {}): Promise { 38 | return this.repoService.use(remote, options, async repo => { 39 | const commit = await this.getCommit(repo, commitSha); 40 | if (!commit) { 41 | return undefined; 42 | } 43 | return toGitCommit(commit); 44 | }); 45 | } 46 | 47 | /** 48 | * @param repo Repository instance 49 | * @param ref Commit SHA, Branch name 50 | */ 51 | public async getCommit(repo: Repository, ref: string | Oid): Promise { 52 | try { 53 | return await repo.getCommit(ref); 54 | } catch { 55 | if (typeof ref !== "string") { 56 | return undefined; 57 | } 58 | try { 59 | return await repo.getReferenceCommit(ref); 60 | } catch (e) { 61 | try { 62 | return await repo.getReferenceCommit(`origin/${ref}`); 63 | } catch { 64 | return undefined; 65 | } 66 | } 67 | } 68 | } 69 | 70 | public async getCommitOrDefault(repo: Repository, ref: string | undefined) { 71 | if (ref) { 72 | return this.getCommit(repo, ref); 73 | } else { 74 | return repo.getReferenceCommit(`origin/master`); 75 | } 76 | } 77 | 78 | public async listCommits( 79 | repo: Repository, 80 | options: ListCommitsOptions, 81 | ): Promise | NotFoundException> { 82 | const walk = repo.createRevWalk(); 83 | 84 | const page = getPage(options.pagination); 85 | const commit = await this.getCommitOrDefault(repo, options.ref); 86 | if (!commit) { 87 | return new NotFoundException(`Couldn't find reference with name ${options.ref}`); 88 | } 89 | walk.push(commit.id()); 90 | 91 | const skip = getPaginationSkip(options.pagination, LIST_COMMIT_PAGE_SIZE); 92 | await walkSkip(walk, skip); 93 | const commits = await walk.getCommits(LIST_COMMIT_PAGE_SIZE); 94 | 95 | let total = skip + LIST_COMMIT_PAGE_SIZE; 96 | 97 | while (true) { 98 | try { 99 | await walk.next(); 100 | total++; 101 | } catch (e) { 102 | break; 103 | } 104 | } 105 | return { 106 | items: commits, 107 | page, 108 | total, 109 | perPage: LIST_COMMIT_PAGE_SIZE, 110 | }; 111 | } 112 | } 113 | 114 | export async function toGitCommit(commit: Commit): Promise { 115 | const [author, committer, parents] = await Promise.all([getAuthor(commit), getCommitter(commit), getParents(commit)]); 116 | return new GitCommit({ 117 | sha: commit.sha(), 118 | message: commit.message(), 119 | author, 120 | committer, 121 | parents, 122 | }); 123 | } 124 | 125 | /** 126 | * Get the list of the parents of the commit 127 | */ 128 | export async function getParents(commit: Commit): Promise { 129 | const parents = await commit.getParents(10); 130 | return parents.map(parent => { 131 | return new GitCommitRef({ sha: parent.sha() }); 132 | }); 133 | } 134 | 135 | export async function getAuthor(commit: Commit): Promise { 136 | const author = await commit.author(); 137 | return getSignature(author); 138 | } 139 | 140 | export async function getCommitter(commit: Commit): Promise { 141 | const committer = await commit.committer(); 142 | return getSignature(committer); 143 | } 144 | 145 | export function getSignature(sig: Signature): GitSignature { 146 | return new GitSignature({ email: sig.email(), name: sig.name(), date: getDateFromTime(sig.when()) }); 147 | } 148 | 149 | export function getDateFromTime(time: Time): Date { 150 | return new Date(time.time() * 1000); 151 | } 152 | 153 | /** 154 | * Try to skip the given number of item in the walk. 155 | * If there is less than ask remaining it will just stop gracfully 156 | */ 157 | async function walkSkip(revwalk: Revwalk, skip: number) { 158 | for (let i = 0; i < skip; i++) { 159 | try { 160 | await revwalk.next(); 161 | } catch { 162 | return; 163 | } 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /src/services/compare/compare.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable, NotFoundException } from "@nestjs/common"; 2 | import { Commit, ConvenientPatch, Diff, Merge, Oid, Repository } from "nodegit"; 3 | 4 | import { Logger } from "../../core"; 5 | import { GitDiff, GitFileDiff, PatchStatus } from "../../dtos"; 6 | import { GitUtils, notUndefined } from "../../utils"; 7 | import { CommitService, toGitCommit } from "../commit"; 8 | import { GitBaseOptions, RepoService } from "../repo"; 9 | 10 | const MAX_COMMIT_PER_DIFF = 250; 11 | const MAX_FILES_PER_DIFF = 300; 12 | 13 | @Injectable() 14 | export class CompareService { 15 | private logger = new Logger(CompareService); 16 | 17 | constructor(private repoService: RepoService, private commitService: CommitService) {} 18 | 19 | public async compare( 20 | remote: string, 21 | base: string, 22 | head: string, 23 | options: GitBaseOptions = {}, 24 | ): Promise { 25 | return this.useCompareRepo(remote, base, head, options, async compareRepo => { 26 | const repo = compareRepo.repo; 27 | const [baseCommit, headCommit] = await Promise.all([ 28 | this.commitService.getCommit(repo, compareRepo.baseRef), 29 | this.commitService.getCommit(repo, compareRepo.headRef), 30 | ]); 31 | if (!baseCommit) { 32 | return new NotFoundException(`Base ${base} was not found`); 33 | } 34 | if (!headCommit) { 35 | return new NotFoundException(`Head ${base} was not found`); 36 | } 37 | 38 | return this.getComparison(repo, baseCommit, headCommit); 39 | }); 40 | } 41 | 42 | public async getMergeBase(repo: Repository, base: Oid, head: Oid): Promise { 43 | try { 44 | const mergeBaseSha = await Merge.base(repo, base, head); 45 | return this.commitService.getCommit(repo, mergeBaseSha.toString()); 46 | } catch (error) { 47 | this.logger.info("Merge base was not found", { error }); 48 | return undefined; 49 | } 50 | } 51 | 52 | public async getComparison( 53 | repo: Repository, 54 | nativeBaseCommit: Commit, 55 | nativeHeadCommit: Commit, 56 | ): Promise { 57 | const [baseCommit, headCommit] = await Promise.all([toGitCommit(nativeBaseCommit), toGitCommit(nativeHeadCommit)]); 58 | 59 | const mergeBase = await this.getMergeBase(repo, nativeBaseCommit.id(), nativeHeadCommit.id()); 60 | if (!mergeBase) { 61 | return new NotFoundException(`Couldn't find a common ancestor for commits`); 62 | } 63 | 64 | const mergeBaseCommit = await toGitCommit(mergeBase); 65 | const files = await this.getFileDiffs(mergeBase, nativeHeadCommit); 66 | const commitIds = await this.listCommitIdsBetween(repo, mergeBase.id(), nativeHeadCommit.id()); 67 | 68 | const commits = await Promise.all( 69 | commitIds.slice(0, MAX_COMMIT_PER_DIFF).map(async x => { 70 | const commit = await this.commitService.getCommit(repo, x); 71 | return commit ? toGitCommit(commit) : undefined; 72 | }), 73 | ); 74 | return new GitDiff({ 75 | baseCommit, 76 | headCommit, 77 | mergeBaseCommit, 78 | totalCommits: commitIds.length, 79 | commits: commits.filter(notUndefined), 80 | files, 81 | }); 82 | } 83 | 84 | public async listCommitIdsBetween(repo: Repository, from: Oid, to: Oid): Promise { 85 | const walk = repo.createRevWalk(); 86 | walk.push(to); 87 | let current: Oid = to; 88 | const commits = []; 89 | while (true) { 90 | current = await walk.next(); 91 | if (current.equal(from)) { 92 | break; 93 | } 94 | commits.push(current); 95 | } 96 | return commits; 97 | } 98 | 99 | public async getFileDiffs(nativeBaseCommit: Commit, nativeHeadCommit: Commit): Promise { 100 | const [baseTree, headTree] = await Promise.all([nativeBaseCommit.getTree(), nativeHeadCommit.getTree()]); 101 | const diff = ((await headTree.diff(baseTree)) as unknown) as Diff; 102 | await diff.findSimilar({ 103 | flags: Diff.FIND.RENAMES, 104 | }); 105 | const patches = await diff.patches(); 106 | 107 | return patches.map(x => toFileDiff(x)).slice(0, MAX_FILES_PER_DIFF); 108 | } 109 | 110 | private async useCompareRepo( 111 | remote: string, 112 | base: string, 113 | head: string, 114 | options: GitBaseOptions, 115 | action: (p: any) => Promise, 116 | ): Promise { 117 | const baseRef = GitUtils.parseRemoteReference(base, remote); 118 | const headRef = GitUtils.parseRemoteReference(head, remote); 119 | 120 | const baseRemote = baseRef.remote; 121 | const headRemote = headRef.remote; 122 | 123 | if (baseRemote !== headRemote) { 124 | return this.repoService.useForCompare( 125 | { 126 | name: "baser", 127 | remote: baseRemote, 128 | }, 129 | { 130 | name: "headr", 131 | remote: headRemote, 132 | }, 133 | options, 134 | repo => 135 | action({ repo, baseRef: `refs/remotes/baser/${baseRef.ref}`, headRef: `refs/remotes/headr/${headRef.ref}` }), 136 | ); 137 | } else { 138 | return this.repoService.use(headRemote, options, repo => 139 | action({ repo, baseRef: baseRef.ref, headRef: headRef.ref }), 140 | ); 141 | } 142 | } 143 | } 144 | 145 | export function toFileDiff(patch: ConvenientPatch): GitFileDiff { 146 | const filename = patch.newFile().path(); 147 | const previousFilename = patch.oldFile().path(); 148 | const stats = patch.lineStats(); 149 | return new GitFileDiff({ 150 | filename, 151 | sha: patch 152 | .newFile() 153 | .id() 154 | .toString(), 155 | status: getPatchStatus(patch), 156 | additions: stats.total_additions, 157 | deletions: stats.total_deletions, 158 | changes: stats.total_additions + stats.total_deletions, 159 | previousFilename: previousFilename !== filename ? previousFilename : undefined, 160 | }); 161 | } 162 | 163 | export function getPatchStatus(patch: ConvenientPatch): PatchStatus { 164 | if (patch.isRenamed()) { 165 | return PatchStatus.Renamed; 166 | } else if (patch.isModified()) { 167 | return PatchStatus.Modified; 168 | } else if (patch.isDeleted()) { 169 | return PatchStatus.Deleted; 170 | } else if (patch.isAdded()) { 171 | return PatchStatus.Added; 172 | } else { 173 | return PatchStatus.Unmodified; 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /src/services/repo/local-repo/local-repo.test.ts: -------------------------------------------------------------------------------- 1 | import nodegit from "nodegit"; 2 | 3 | import { RepoAuth } from "../../../core"; 4 | import { Deferred, delay } from "../../../utils"; 5 | import { LocalRepo } from "./local-repo"; 6 | 7 | const origin = { 8 | name: "origin", 9 | remote: "example.com/git-rest-api.git", 10 | }; 11 | 12 | describe("LocalRepo", () => { 13 | let repo: LocalRepo; 14 | 15 | const fsSpy = { 16 | exists: jest.fn(), 17 | rm: jest.fn(), 18 | }; 19 | const repoIndexSpy = { 20 | markRepoAsOpened: jest.fn(), 21 | markRepoAsFetched: jest.fn(), 22 | }; 23 | 24 | const onDestroy = jest.fn(); 25 | 26 | const repoSpy = { 27 | cleanup: jest.fn(), 28 | fetchAll: jest.fn(() => Promise.resolve()), 29 | }; 30 | const MockRepository = { 31 | open: jest.fn(() => Promise.resolve(repoSpy)), 32 | init: jest.fn(() => Promise.resolve(repoSpy)), 33 | }; 34 | 35 | const MockRemote = { 36 | create: jest.fn(), 37 | }; 38 | 39 | beforeEach(() => { 40 | jest.clearAllMocks(); 41 | 42 | nodegit.Repository = MockRepository as any; 43 | nodegit.Remote = MockRemote as any; 44 | 45 | repo = new LocalRepo("foo", fsSpy as any, repoIndexSpy as any); 46 | repo.onDestroy.subscribe(onDestroy); 47 | }); 48 | 49 | afterEach(() => { 50 | if (repo) { 51 | repo.dispose(); 52 | } 53 | }); 54 | 55 | describe("init the repo", () => { 56 | it("opens the existing one if the path exists", async () => { 57 | fsSpy.exists.mockResolvedValue(true); 58 | await repo.init([origin]); 59 | expect(MockRepository.open).toHaveBeenCalledTimes(1); 60 | expect(MockRepository.open).toHaveBeenCalledWith("foo"); 61 | expect(MockRepository.init).not.toHaveBeenCalled(); 62 | 63 | expect(fsSpy.rm).not.toHaveBeenCalled(); 64 | }); 65 | 66 | it("init the repo if the path doesn't exists", async () => { 67 | fsSpy.exists.mockResolvedValue(false); 68 | await repo.init([origin]); 69 | expect(MockRepository.init).toHaveBeenCalledTimes(1); 70 | expect(MockRepository.init).toHaveBeenCalledWith("foo", 1); 71 | expect(MockRepository.open).not.toHaveBeenCalled(); 72 | 73 | expect(MockRemote.create).toHaveBeenCalledTimes(1); 74 | expect(MockRemote.create).toHaveBeenCalledWith(repoSpy, "origin", "https://example.com/git-rest-api.git"); 75 | }); 76 | 77 | it("delete then reinit the repo if it fails to open when the path exists", async () => { 78 | fsSpy.exists.mockResolvedValue(true); 79 | MockRepository.open.mockRejectedValueOnce(new Error("Failed to open repo")); 80 | await repo.init([origin]); 81 | expect(MockRepository.open).toHaveBeenCalledTimes(1); 82 | expect(MockRepository.open).toHaveBeenCalledWith("foo"); 83 | 84 | expect(fsSpy.rm).toHaveBeenCalledTimes(1); 85 | expect(fsSpy.rm).toHaveBeenCalledWith("foo"); 86 | 87 | expect(MockRepository.init).toHaveBeenCalledTimes(1); 88 | expect(MockRepository.init).toHaveBeenCalledWith("foo", 1); 89 | 90 | expect(MockRemote.create).toHaveBeenCalledTimes(1); 91 | expect(MockRemote.create).toHaveBeenCalledWith(repoSpy, "origin", "https://example.com/git-rest-api.git"); 92 | }); 93 | 94 | it("calling init again doesn't do anything", async () => { 95 | fsSpy.exists.mockResolvedValue(true); 96 | const init1 = await repo.init([origin]); 97 | await delay(); 98 | const init2 = repo.init([origin]); 99 | await init1; 100 | await init2; 101 | expect(MockRepository.open).toHaveBeenCalledTimes(1); 102 | expect(MockRepository.init).not.toHaveBeenCalled(); 103 | 104 | await repo.init([origin]); 105 | expect(MockRepository.open).toHaveBeenCalledTimes(1); 106 | expect(MockRepository.init).not.toHaveBeenCalled(); 107 | expect(fsSpy.rm).not.toHaveBeenCalled(); 108 | }); 109 | }); 110 | 111 | describe("Update and use", () => { 112 | beforeEach(async () => { 113 | fsSpy.exists.mockResolvedValue(true); 114 | await repo.init([origin]); 115 | }); 116 | 117 | it("use the repo", async () => { 118 | const response = await repo.use(async r => { 119 | expect(r).toBe(repoSpy); 120 | return "My-result"; 121 | }); 122 | 123 | expect(response).toEqual("My-result"); 124 | }); 125 | 126 | it("update the repo", async () => { 127 | const options = { 128 | auth: new RepoAuth(), 129 | }; 130 | await repo.update(options); 131 | expect(repoSpy.fetchAll).toHaveBeenCalledTimes(1); 132 | }); 133 | 134 | it("doesn't trigger multiple updates if one is already in progress", async () => { 135 | let resolve: () => void; 136 | const fetchPromise = new Promise(r => (resolve = r)); 137 | repoSpy.fetchAll.mockImplementation(() => fetchPromise); 138 | const update1 = repo.update(); 139 | const update2 = repo.update(); 140 | await delay(); 141 | const update3 = repo.update(); 142 | resolve!(); 143 | await Promise.all([update1, update2, update3]); 144 | expect(repoSpy.fetchAll).toHaveBeenCalledTimes(1); 145 | }); 146 | 147 | it("should wait for uses to complete before updating", async () => { 148 | const use1Deferer = new Deferred(); 149 | const use2Deferer = new Deferred(); 150 | const use3Deferer = new Deferred(); 151 | 152 | const use1 = repo.use(() => use1Deferer.promise); 153 | const use2 = repo.use(() => use2Deferer.promise); 154 | 155 | const update = repo.update(); 156 | 157 | const use3 = repo.use(() => use3Deferer.promise); 158 | 159 | expect(repoSpy.fetchAll).not.toHaveBeenCalled(); 160 | 161 | use1Deferer.resolve(); 162 | use2Deferer.resolve(); 163 | await use1; 164 | await use2; 165 | await update; 166 | expect(repoSpy.fetchAll).toHaveBeenCalledTimes(1); 167 | use3Deferer.resolve(); 168 | await use3; 169 | }); 170 | }); 171 | 172 | describe("dispose of the repo when nothing is using it", () => { 173 | beforeEach(() => { 174 | repo.dispose(); 175 | jest.useFakeTimers(); 176 | jest.clearAllMocks(); 177 | 178 | repo = new LocalRepo("foo", fsSpy as any, repoIndexSpy as any); 179 | repo.onDestroy.subscribe(onDestroy); 180 | }); 181 | 182 | afterEach(() => { 183 | jest.clearAllTimers(); 184 | jest.useRealTimers(); 185 | }); 186 | 187 | it("does nothing if the repo wasn't opened", () => { 188 | expect(onDestroy).not.toHaveBeenCalled(); 189 | repo.unref(); 190 | expect(onDestroy).not.toHaveBeenCalled(); 191 | jest.advanceTimersByTime(200); 192 | expect(repoSpy.cleanup).not.toHaveBeenCalled(); 193 | expect(onDestroy).toHaveBeenCalledTimes(1); 194 | }); 195 | 196 | it("destroy the repo if it was opened", async () => { 197 | fsSpy.exists.mockResolvedValue(true); 198 | expect(onDestroy).not.toHaveBeenCalled(); 199 | await repo.init([origin]); 200 | repo.unref(); 201 | expect(onDestroy).not.toHaveBeenCalled(); 202 | jest.advanceTimersByTime(200); 203 | expect(repoSpy.cleanup).toHaveBeenCalledTimes(1); 204 | expect(onDestroy).toHaveBeenCalledTimes(1); 205 | }); 206 | 207 | it("counts the refs", async () => { 208 | fsSpy.exists.mockResolvedValue(true); 209 | expect(onDestroy).not.toHaveBeenCalled(); 210 | repo.ref(); 211 | repo.ref(); 212 | await repo.init([origin]); 213 | 214 | // Remove ref 1/3 215 | repo.unref(); 216 | jest.advanceTimersByTime(200); 217 | expect(onDestroy).not.toHaveBeenCalled(); 218 | 219 | // Remove ref 2/3 220 | repo.unref(); 221 | jest.advanceTimersByTime(200); 222 | expect(onDestroy).not.toHaveBeenCalled(); 223 | 224 | // Remove ref 3/3 225 | repo.unref(); 226 | jest.advanceTimersByTime(200); 227 | expect(repoSpy.cleanup).toHaveBeenCalledTimes(1); 228 | expect(onDestroy).toHaveBeenCalledTimes(1); 229 | }); 230 | 231 | it("shouldn't delete if repo is reused within the timeout period", async () => { 232 | fsSpy.exists.mockResolvedValue(true); 233 | expect(onDestroy).not.toHaveBeenCalled(); 234 | await repo.init([origin]); 235 | 236 | // Remove ref 237 | repo.unref(); 238 | jest.advanceTimersByTime(50); 239 | expect(onDestroy).not.toHaveBeenCalled(); 240 | 241 | // Other ref coming in 242 | repo.ref(); 243 | jest.advanceTimersByTime(200); 244 | expect(onDestroy).not.toHaveBeenCalled(); 245 | 246 | // Remove other ref 247 | repo.unref(); 248 | jest.advanceTimersByTime(200); 249 | expect(repoSpy.cleanup).toHaveBeenCalledTimes(1); 250 | expect(onDestroy).toHaveBeenCalledTimes(1); 251 | }); 252 | }); 253 | }); 254 | -------------------------------------------------------------------------------- /src/controllers/content/__snapshots__/content.controller.e2e.ts.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`Test content controller for path '/repos/github.com%2Ftest-repo-billy%2Fgit-api-tests/contents' 1`] = ` 4 | Object { 5 | "dirs": Array [ 6 | Object { 7 | "name": "dir1", 8 | "path": "dir1", 9 | "sha": "b638a8a4a9f44184a3a430988a9c5ef383bad364", 10 | "size": 0, 11 | "type": "dir", 12 | }, 13 | ], 14 | "files": Array [ 15 | Object { 16 | "content": "IyBMb2dzCmxvZ3MKKi5sb2cKbnBtLWRlYnVnLmxvZyoKeWFybi1kZWJ1Zy5sb2cqCnlhcm4tZXJyb3IubG9nKgoKIyBSdW50aW1lIGRhdGEKcGlkcwoqLnBpZAoqLnNlZWQKKi5waWQubG9jawoKIyBEaXJlY3RvcnkgZm9yIGluc3RydW1lbnRlZCBsaWJzIGdlbmVyYXRlZCBieSBqc2NvdmVyYWdlL0pTQ292ZXIKbGliLWNvdgoKIyBDb3ZlcmFnZSBkaXJlY3RvcnkgdXNlZCBieSB0b29scyBsaWtlIGlzdGFuYnVsCmNvdmVyYWdlCgojIG55YyB0ZXN0IGNvdmVyYWdlCi5ueWNfb3V0cHV0CgojIEdydW50IGludGVybWVkaWF0ZSBzdG9yYWdlIChodHRwOi8vZ3J1bnRqcy5jb20vY3JlYXRpbmctcGx1Z2lucyNzdG9yaW5nLXRhc2stZmlsZXMpCi5ncnVudAoKIyBCb3dlciBkZXBlbmRlbmN5IGRpcmVjdG9yeSAoaHR0cHM6Ly9ib3dlci5pby8pCmJvd2VyX2NvbXBvbmVudHMKCiMgbm9kZS13YWYgY29uZmlndXJhdGlvbgoubG9jay13c2NyaXB0CgojIENvbXBpbGVkIGJpbmFyeSBhZGRvbnMgKGh0dHBzOi8vbm9kZWpzLm9yZy9hcGkvYWRkb25zLmh0bWwpCmJ1aWxkL1JlbGVhc2UKCiMgRGVwZW5kZW5jeSBkaXJlY3Rvcmllcwpub2RlX21vZHVsZXMvCmpzcG1fcGFja2FnZXMvCgojIFR5cGVTY3JpcHQgdjEgZGVjbGFyYXRpb24gZmlsZXMKdHlwaW5ncy8KCiMgT3B0aW9uYWwgbnBtIGNhY2hlIGRpcmVjdG9yeQoubnBtCgojIE9wdGlvbmFsIGVzbGludCBjYWNoZQouZXNsaW50Y2FjaGUKCiMgT3B0aW9uYWwgUkVQTCBoaXN0b3J5Ci5ub2RlX3JlcGxfaGlzdG9yeQoKIyBPdXRwdXQgb2YgJ25wbSBwYWNrJwoqLnRnegoKIyBZYXJuIEludGVncml0eSBmaWxlCi55YXJuLWludGVncml0eQoKIyBkb3RlbnYgZW52aXJvbm1lbnQgdmFyaWFibGVzIGZpbGUKLmVudgoKIyBuZXh0LmpzIGJ1aWxkIG91dHB1dAoubmV4dAo=", 17 | "encoding": "base64", 18 | "name": ".gitignore", 19 | "path": ".gitignore", 20 | "sha": "ad46b30886fa350c1f59761b100e5e4b01f9a7ec", 21 | "size": 914, 22 | "type": "file", 23 | }, 24 | Object { 25 | "content": "TUlUIExpY2Vuc2UKCkNvcHlyaWdodCAoYykgMjAxOSB0ZXN0LXJlcG8tYmlsbHkKClBlcm1pc3Npb24gaXMgaGVyZWJ5IGdyYW50ZWQsIGZyZWUgb2YgY2hhcmdlLCB0byBhbnkgcGVyc29uIG9idGFpbmluZyBhIGNvcHkKb2YgdGhpcyBzb2Z0d2FyZSBhbmQgYXNzb2NpYXRlZCBkb2N1bWVudGF0aW9uIGZpbGVzICh0aGUgIlNvZnR3YXJlIiksIHRvIGRlYWwKaW4gdGhlIFNvZnR3YXJlIHdpdGhvdXQgcmVzdHJpY3Rpb24sIGluY2x1ZGluZyB3aXRob3V0IGxpbWl0YXRpb24gdGhlIHJpZ2h0cwp0byB1c2UsIGNvcHksIG1vZGlmeSwgbWVyZ2UsIHB1Ymxpc2gsIGRpc3RyaWJ1dGUsIHN1YmxpY2Vuc2UsIGFuZC9vciBzZWxsCmNvcGllcyBvZiB0aGUgU29mdHdhcmUsIGFuZCB0byBwZXJtaXQgcGVyc29ucyB0byB3aG9tIHRoZSBTb2Z0d2FyZSBpcwpmdXJuaXNoZWQgdG8gZG8gc28sIHN1YmplY3QgdG8gdGhlIGZvbGxvd2luZyBjb25kaXRpb25zOgoKVGhlIGFib3ZlIGNvcHlyaWdodCBub3RpY2UgYW5kIHRoaXMgcGVybWlzc2lvbiBub3RpY2Ugc2hhbGwgYmUgaW5jbHVkZWQgaW4gYWxsCmNvcGllcyBvciBzdWJzdGFudGlhbCBwb3J0aW9ucyBvZiB0aGUgU29mdHdhcmUuCgpUSEUgU09GVFdBUkUgSVMgUFJPVklERUQgIkFTIElTIiwgV0lUSE9VVCBXQVJSQU5UWSBPRiBBTlkgS0lORCwgRVhQUkVTUyBPUgpJTVBMSUVELCBJTkNMVURJTkcgQlVUIE5PVCBMSU1JVEVEIFRPIFRIRSBXQVJSQU5USUVTIE9GIE1FUkNIQU5UQUJJTElUWSwKRklUTkVTUyBGT1IgQSBQQVJUSUNVTEFSIFBVUlBPU0UgQU5EIE5PTklORlJJTkdFTUVOVC4gSU4gTk8gRVZFTlQgU0hBTEwgVEhFCkFVVEhPUlMgT1IgQ09QWVJJR0hUIEhPTERFUlMgQkUgTElBQkxFIEZPUiBBTlkgQ0xBSU0sIERBTUFHRVMgT1IgT1RIRVIKTElBQklMSVRZLCBXSEVUSEVSIElOIEFOIEFDVElPTiBPRiBDT05UUkFDVCwgVE9SVCBPUiBPVEhFUldJU0UsIEFSSVNJTkcgRlJPTSwKT1VUIE9GIE9SIElOIENPTk5FQ1RJT04gV0lUSCBUSEUgU09GVFdBUkUgT1IgVEhFIFVTRSBPUiBPVEhFUiBERUFMSU5HUyBJTiBUSEUKU09GVFdBUkUuCg==", 26 | "encoding": "base64", 27 | "name": "LICENSE", 28 | "path": "LICENSE", 29 | "sha": "98023418cdb98210a5f71ea74ec557dbbd8f0e83", 30 | "size": 1072, 31 | "type": "file", 32 | }, 33 | Object { 34 | "content": "IyBnaXQtYXBpLXRlc3RzClJlcG8gdXNlZCBmb3IgaW50ZWdyYXRpb24gdGVzdGluZyBvZiB0aGUgZ2l0LXRlc3QtYXBpIHByb2plY3QK", 35 | "encoding": "base64", 36 | "name": "README.md", 37 | "path": "README.md", 38 | "sha": "b5fd37e731f1e7931da42484ae0290554cb42c0f", 39 | "size": 78, 40 | "type": "file", 41 | }, 42 | ], 43 | "submodules": Array [], 44 | } 45 | `; 46 | 47 | exports[`Test content controller for path '/repos/github.com%2Ftest-repo-billy%2Fgit-api-tests/contents/' 1`] = ` 48 | Object { 49 | "dirs": Array [ 50 | Object { 51 | "name": "dir1", 52 | "path": "dir1", 53 | "sha": "b638a8a4a9f44184a3a430988a9c5ef383bad364", 54 | "size": 0, 55 | "type": "dir", 56 | }, 57 | ], 58 | "files": Array [ 59 | Object { 60 | "content": "IyBMb2dzCmxvZ3MKKi5sb2cKbnBtLWRlYnVnLmxvZyoKeWFybi1kZWJ1Zy5sb2cqCnlhcm4tZXJyb3IubG9nKgoKIyBSdW50aW1lIGRhdGEKcGlkcwoqLnBpZAoqLnNlZWQKKi5waWQubG9jawoKIyBEaXJlY3RvcnkgZm9yIGluc3RydW1lbnRlZCBsaWJzIGdlbmVyYXRlZCBieSBqc2NvdmVyYWdlL0pTQ292ZXIKbGliLWNvdgoKIyBDb3ZlcmFnZSBkaXJlY3RvcnkgdXNlZCBieSB0b29scyBsaWtlIGlzdGFuYnVsCmNvdmVyYWdlCgojIG55YyB0ZXN0IGNvdmVyYWdlCi5ueWNfb3V0cHV0CgojIEdydW50IGludGVybWVkaWF0ZSBzdG9yYWdlIChodHRwOi8vZ3J1bnRqcy5jb20vY3JlYXRpbmctcGx1Z2lucyNzdG9yaW5nLXRhc2stZmlsZXMpCi5ncnVudAoKIyBCb3dlciBkZXBlbmRlbmN5IGRpcmVjdG9yeSAoaHR0cHM6Ly9ib3dlci5pby8pCmJvd2VyX2NvbXBvbmVudHMKCiMgbm9kZS13YWYgY29uZmlndXJhdGlvbgoubG9jay13c2NyaXB0CgojIENvbXBpbGVkIGJpbmFyeSBhZGRvbnMgKGh0dHBzOi8vbm9kZWpzLm9yZy9hcGkvYWRkb25zLmh0bWwpCmJ1aWxkL1JlbGVhc2UKCiMgRGVwZW5kZW5jeSBkaXJlY3Rvcmllcwpub2RlX21vZHVsZXMvCmpzcG1fcGFja2FnZXMvCgojIFR5cGVTY3JpcHQgdjEgZGVjbGFyYXRpb24gZmlsZXMKdHlwaW5ncy8KCiMgT3B0aW9uYWwgbnBtIGNhY2hlIGRpcmVjdG9yeQoubnBtCgojIE9wdGlvbmFsIGVzbGludCBjYWNoZQouZXNsaW50Y2FjaGUKCiMgT3B0aW9uYWwgUkVQTCBoaXN0b3J5Ci5ub2RlX3JlcGxfaGlzdG9yeQoKIyBPdXRwdXQgb2YgJ25wbSBwYWNrJwoqLnRnegoKIyBZYXJuIEludGVncml0eSBmaWxlCi55YXJuLWludGVncml0eQoKIyBkb3RlbnYgZW52aXJvbm1lbnQgdmFyaWFibGVzIGZpbGUKLmVudgoKIyBuZXh0LmpzIGJ1aWxkIG91dHB1dAoubmV4dAo=", 61 | "encoding": "base64", 62 | "name": ".gitignore", 63 | "path": ".gitignore", 64 | "sha": "ad46b30886fa350c1f59761b100e5e4b01f9a7ec", 65 | "size": 914, 66 | "type": "file", 67 | }, 68 | Object { 69 | "content": "TUlUIExpY2Vuc2UKCkNvcHlyaWdodCAoYykgMjAxOSB0ZXN0LXJlcG8tYmlsbHkKClBlcm1pc3Npb24gaXMgaGVyZWJ5IGdyYW50ZWQsIGZyZWUgb2YgY2hhcmdlLCB0byBhbnkgcGVyc29uIG9idGFpbmluZyBhIGNvcHkKb2YgdGhpcyBzb2Z0d2FyZSBhbmQgYXNzb2NpYXRlZCBkb2N1bWVudGF0aW9uIGZpbGVzICh0aGUgIlNvZnR3YXJlIiksIHRvIGRlYWwKaW4gdGhlIFNvZnR3YXJlIHdpdGhvdXQgcmVzdHJpY3Rpb24sIGluY2x1ZGluZyB3aXRob3V0IGxpbWl0YXRpb24gdGhlIHJpZ2h0cwp0byB1c2UsIGNvcHksIG1vZGlmeSwgbWVyZ2UsIHB1Ymxpc2gsIGRpc3RyaWJ1dGUsIHN1YmxpY2Vuc2UsIGFuZC9vciBzZWxsCmNvcGllcyBvZiB0aGUgU29mdHdhcmUsIGFuZCB0byBwZXJtaXQgcGVyc29ucyB0byB3aG9tIHRoZSBTb2Z0d2FyZSBpcwpmdXJuaXNoZWQgdG8gZG8gc28sIHN1YmplY3QgdG8gdGhlIGZvbGxvd2luZyBjb25kaXRpb25zOgoKVGhlIGFib3ZlIGNvcHlyaWdodCBub3RpY2UgYW5kIHRoaXMgcGVybWlzc2lvbiBub3RpY2Ugc2hhbGwgYmUgaW5jbHVkZWQgaW4gYWxsCmNvcGllcyBvciBzdWJzdGFudGlhbCBwb3J0aW9ucyBvZiB0aGUgU29mdHdhcmUuCgpUSEUgU09GVFdBUkUgSVMgUFJPVklERUQgIkFTIElTIiwgV0lUSE9VVCBXQVJSQU5UWSBPRiBBTlkgS0lORCwgRVhQUkVTUyBPUgpJTVBMSUVELCBJTkNMVURJTkcgQlVUIE5PVCBMSU1JVEVEIFRPIFRIRSBXQVJSQU5USUVTIE9GIE1FUkNIQU5UQUJJTElUWSwKRklUTkVTUyBGT1IgQSBQQVJUSUNVTEFSIFBVUlBPU0UgQU5EIE5PTklORlJJTkdFTUVOVC4gSU4gTk8gRVZFTlQgU0hBTEwgVEhFCkFVVEhPUlMgT1IgQ09QWVJJR0hUIEhPTERFUlMgQkUgTElBQkxFIEZPUiBBTlkgQ0xBSU0sIERBTUFHRVMgT1IgT1RIRVIKTElBQklMSVRZLCBXSEVUSEVSIElOIEFOIEFDVElPTiBPRiBDT05UUkFDVCwgVE9SVCBPUiBPVEhFUldJU0UsIEFSSVNJTkcgRlJPTSwKT1VUIE9GIE9SIElOIENPTk5FQ1RJT04gV0lUSCBUSEUgU09GVFdBUkUgT1IgVEhFIFVTRSBPUiBPVEhFUiBERUFMSU5HUyBJTiBUSEUKU09GVFdBUkUuCg==", 70 | "encoding": "base64", 71 | "name": "LICENSE", 72 | "path": "LICENSE", 73 | "sha": "98023418cdb98210a5f71ea74ec557dbbd8f0e83", 74 | "size": 1072, 75 | "type": "file", 76 | }, 77 | Object { 78 | "content": "IyBnaXQtYXBpLXRlc3RzClJlcG8gdXNlZCBmb3IgaW50ZWdyYXRpb24gdGVzdGluZyBvZiB0aGUgZ2l0LXRlc3QtYXBpIHByb2plY3QK", 79 | "encoding": "base64", 80 | "name": "README.md", 81 | "path": "README.md", 82 | "sha": "b5fd37e731f1e7931da42484ae0290554cb42c0f", 83 | "size": 78, 84 | "type": "file", 85 | }, 86 | ], 87 | "submodules": Array [], 88 | } 89 | `; 90 | 91 | exports[`Test content controller for path '/repos/github.com%2Ftest-repo-billy%2Fgit-api-tests/contents/README.md' 1`] = ` 92 | Object { 93 | "dirs": Array [], 94 | "files": Array [ 95 | Object { 96 | "content": "IyBnaXQtYXBpLXRlc3RzClJlcG8gdXNlZCBmb3IgaW50ZWdyYXRpb24gdGVzdGluZyBvZiB0aGUgZ2l0LXRlc3QtYXBpIHByb2plY3QK", 97 | "encoding": "base64", 98 | "name": "README.md", 99 | "path": "README.md", 100 | "sha": "b5fd37e731f1e7931da42484ae0290554cb42c0f", 101 | "size": 78, 102 | "type": "file", 103 | }, 104 | ], 105 | "submodules": Array [], 106 | } 107 | `; 108 | 109 | exports[`Test content controller for path '/repos/github.com%2Ftest-repo-billy%2Fgit-api-tests/contents/dir1' 1`] = ` 110 | Object { 111 | "dirs": Array [ 112 | Object { 113 | "name": "dir2", 114 | "path": "dir1/dir2", 115 | "sha": "483221c9d8371862bdb2c5d452130ab5ca0534a3", 116 | "size": 0, 117 | "type": "dir", 118 | }, 119 | ], 120 | "files": Array [ 121 | Object { 122 | "content": "ZmlsZUEK", 123 | "encoding": "base64", 124 | "name": "fileA.txt", 125 | "path": "dir1/fileA.txt", 126 | "sha": "ab47708c98ac88bbdf3ca75f4730d86a84f702a2", 127 | "size": 6, 128 | "type": "file", 129 | }, 130 | ], 131 | "submodules": Array [], 132 | } 133 | `; 134 | 135 | exports[`Test content controller for path '/repos/github.com%2Ftest-repo-billy%2Fgit-api-tests/contents/dir1?recursive=true' 1`] = ` 136 | Object { 137 | "dirs": Array [ 138 | Object { 139 | "name": "dir2", 140 | "path": "dir1/dir2", 141 | "sha": "483221c9d8371862bdb2c5d452130ab5ca0534a3", 142 | "size": 0, 143 | "type": "dir", 144 | }, 145 | ], 146 | "files": Array [ 147 | Object { 148 | "content": "ZmlsZUEK", 149 | "encoding": "base64", 150 | "name": "fileA.txt", 151 | "path": "dir1/fileA.txt", 152 | "sha": "ab47708c98ac88bbdf3ca75f4730d86a84f702a2", 153 | "size": 6, 154 | "type": "file", 155 | }, 156 | Object { 157 | "content": "ZmlsZUIK", 158 | "encoding": "base64", 159 | "name": "fileB.txt", 160 | "path": "dir1/dir2/fileB.txt", 161 | "sha": "78ed112c991c8abeba325c039a398ba626c425ab", 162 | "size": 6, 163 | "type": "file", 164 | }, 165 | ], 166 | "submodules": Array [], 167 | } 168 | `; 169 | 170 | exports[`Test content controller for path '/repos/github.com%2Ftest-repo-billy%2Fgit-api-tests/contents?recursive=true' 1`] = ` 171 | Object { 172 | "dirs": Array [ 173 | Object { 174 | "name": "dir1", 175 | "path": "dir1", 176 | "sha": "b638a8a4a9f44184a3a430988a9c5ef383bad364", 177 | "size": 0, 178 | "type": "dir", 179 | }, 180 | Object { 181 | "name": "dir2", 182 | "path": "dir1/dir2", 183 | "sha": "483221c9d8371862bdb2c5d452130ab5ca0534a3", 184 | "size": 0, 185 | "type": "dir", 186 | }, 187 | ], 188 | "files": Array [ 189 | Object { 190 | "content": "IyBMb2dzCmxvZ3MKKi5sb2cKbnBtLWRlYnVnLmxvZyoKeWFybi1kZWJ1Zy5sb2cqCnlhcm4tZXJyb3IubG9nKgoKIyBSdW50aW1lIGRhdGEKcGlkcwoqLnBpZAoqLnNlZWQKKi5waWQubG9jawoKIyBEaXJlY3RvcnkgZm9yIGluc3RydW1lbnRlZCBsaWJzIGdlbmVyYXRlZCBieSBqc2NvdmVyYWdlL0pTQ292ZXIKbGliLWNvdgoKIyBDb3ZlcmFnZSBkaXJlY3RvcnkgdXNlZCBieSB0b29scyBsaWtlIGlzdGFuYnVsCmNvdmVyYWdlCgojIG55YyB0ZXN0IGNvdmVyYWdlCi5ueWNfb3V0cHV0CgojIEdydW50IGludGVybWVkaWF0ZSBzdG9yYWdlIChodHRwOi8vZ3J1bnRqcy5jb20vY3JlYXRpbmctcGx1Z2lucyNzdG9yaW5nLXRhc2stZmlsZXMpCi5ncnVudAoKIyBCb3dlciBkZXBlbmRlbmN5IGRpcmVjdG9yeSAoaHR0cHM6Ly9ib3dlci5pby8pCmJvd2VyX2NvbXBvbmVudHMKCiMgbm9kZS13YWYgY29uZmlndXJhdGlvbgoubG9jay13c2NyaXB0CgojIENvbXBpbGVkIGJpbmFyeSBhZGRvbnMgKGh0dHBzOi8vbm9kZWpzLm9yZy9hcGkvYWRkb25zLmh0bWwpCmJ1aWxkL1JlbGVhc2UKCiMgRGVwZW5kZW5jeSBkaXJlY3Rvcmllcwpub2RlX21vZHVsZXMvCmpzcG1fcGFja2FnZXMvCgojIFR5cGVTY3JpcHQgdjEgZGVjbGFyYXRpb24gZmlsZXMKdHlwaW5ncy8KCiMgT3B0aW9uYWwgbnBtIGNhY2hlIGRpcmVjdG9yeQoubnBtCgojIE9wdGlvbmFsIGVzbGludCBjYWNoZQouZXNsaW50Y2FjaGUKCiMgT3B0aW9uYWwgUkVQTCBoaXN0b3J5Ci5ub2RlX3JlcGxfaGlzdG9yeQoKIyBPdXRwdXQgb2YgJ25wbSBwYWNrJwoqLnRnegoKIyBZYXJuIEludGVncml0eSBmaWxlCi55YXJuLWludGVncml0eQoKIyBkb3RlbnYgZW52aXJvbm1lbnQgdmFyaWFibGVzIGZpbGUKLmVudgoKIyBuZXh0LmpzIGJ1aWxkIG91dHB1dAoubmV4dAo=", 191 | "encoding": "base64", 192 | "name": ".gitignore", 193 | "path": ".gitignore", 194 | "sha": "ad46b30886fa350c1f59761b100e5e4b01f9a7ec", 195 | "size": 914, 196 | "type": "file", 197 | }, 198 | Object { 199 | "content": "TUlUIExpY2Vuc2UKCkNvcHlyaWdodCAoYykgMjAxOSB0ZXN0LXJlcG8tYmlsbHkKClBlcm1pc3Npb24gaXMgaGVyZWJ5IGdyYW50ZWQsIGZyZWUgb2YgY2hhcmdlLCB0byBhbnkgcGVyc29uIG9idGFpbmluZyBhIGNvcHkKb2YgdGhpcyBzb2Z0d2FyZSBhbmQgYXNzb2NpYXRlZCBkb2N1bWVudGF0aW9uIGZpbGVzICh0aGUgIlNvZnR3YXJlIiksIHRvIGRlYWwKaW4gdGhlIFNvZnR3YXJlIHdpdGhvdXQgcmVzdHJpY3Rpb24sIGluY2x1ZGluZyB3aXRob3V0IGxpbWl0YXRpb24gdGhlIHJpZ2h0cwp0byB1c2UsIGNvcHksIG1vZGlmeSwgbWVyZ2UsIHB1Ymxpc2gsIGRpc3RyaWJ1dGUsIHN1YmxpY2Vuc2UsIGFuZC9vciBzZWxsCmNvcGllcyBvZiB0aGUgU29mdHdhcmUsIGFuZCB0byBwZXJtaXQgcGVyc29ucyB0byB3aG9tIHRoZSBTb2Z0d2FyZSBpcwpmdXJuaXNoZWQgdG8gZG8gc28sIHN1YmplY3QgdG8gdGhlIGZvbGxvd2luZyBjb25kaXRpb25zOgoKVGhlIGFib3ZlIGNvcHlyaWdodCBub3RpY2UgYW5kIHRoaXMgcGVybWlzc2lvbiBub3RpY2Ugc2hhbGwgYmUgaW5jbHVkZWQgaW4gYWxsCmNvcGllcyBvciBzdWJzdGFudGlhbCBwb3J0aW9ucyBvZiB0aGUgU29mdHdhcmUuCgpUSEUgU09GVFdBUkUgSVMgUFJPVklERUQgIkFTIElTIiwgV0lUSE9VVCBXQVJSQU5UWSBPRiBBTlkgS0lORCwgRVhQUkVTUyBPUgpJTVBMSUVELCBJTkNMVURJTkcgQlVUIE5PVCBMSU1JVEVEIFRPIFRIRSBXQVJSQU5USUVTIE9GIE1FUkNIQU5UQUJJTElUWSwKRklUTkVTUyBGT1IgQSBQQVJUSUNVTEFSIFBVUlBPU0UgQU5EIE5PTklORlJJTkdFTUVOVC4gSU4gTk8gRVZFTlQgU0hBTEwgVEhFCkFVVEhPUlMgT1IgQ09QWVJJR0hUIEhPTERFUlMgQkUgTElBQkxFIEZPUiBBTlkgQ0xBSU0sIERBTUFHRVMgT1IgT1RIRVIKTElBQklMSVRZLCBXSEVUSEVSIElOIEFOIEFDVElPTiBPRiBDT05UUkFDVCwgVE9SVCBPUiBPVEhFUldJU0UsIEFSSVNJTkcgRlJPTSwKT1VUIE9GIE9SIElOIENPTk5FQ1RJT04gV0lUSCBUSEUgU09GVFdBUkUgT1IgVEhFIFVTRSBPUiBPVEhFUiBERUFMSU5HUyBJTiBUSEUKU09GVFdBUkUuCg==", 200 | "encoding": "base64", 201 | "name": "LICENSE", 202 | "path": "LICENSE", 203 | "sha": "98023418cdb98210a5f71ea74ec557dbbd8f0e83", 204 | "size": 1072, 205 | "type": "file", 206 | }, 207 | Object { 208 | "content": "IyBnaXQtYXBpLXRlc3RzClJlcG8gdXNlZCBmb3IgaW50ZWdyYXRpb24gdGVzdGluZyBvZiB0aGUgZ2l0LXRlc3QtYXBpIHByb2plY3QK", 209 | "encoding": "base64", 210 | "name": "README.md", 211 | "path": "README.md", 212 | "sha": "b5fd37e731f1e7931da42484ae0290554cb42c0f", 213 | "size": 78, 214 | "type": "file", 215 | }, 216 | Object { 217 | "content": "ZmlsZUEK", 218 | "encoding": "base64", 219 | "name": "fileA.txt", 220 | "path": "dir1/fileA.txt", 221 | "sha": "ab47708c98ac88bbdf3ca75f4730d86a84f702a2", 222 | "size": 6, 223 | "type": "file", 224 | }, 225 | Object { 226 | "content": "ZmlsZUIK", 227 | "encoding": "base64", 228 | "name": "fileB.txt", 229 | "path": "dir1/dir2/fileB.txt", 230 | "sha": "78ed112c991c8abeba325c039a398ba626c425ab", 231 | "size": 6, 232 | "type": "file", 233 | }, 234 | ], 235 | "submodules": Array [], 236 | } 237 | `; 238 | -------------------------------------------------------------------------------- /swagger-spec.json: -------------------------------------------------------------------------------- 1 | { 2 | "swagger": "2.0", 3 | "info": { 4 | "description": "Rest api to run operation on git repositories", 5 | "version": "1.0", 6 | "title": "GIT Rest API" 7 | }, 8 | "basePath": "/", 9 | "tags": [], 10 | "schemes": [ 11 | "http", 12 | "https" 13 | ], 14 | "paths": { 15 | "/health/alive": { 16 | "get": { 17 | "summary": "Check alive", 18 | "operationId": "health_checkAlive", 19 | "responses": { 20 | "200": { 21 | "description": "" 22 | } 23 | }, 24 | "produces": [ 25 | "application/json" 26 | ], 27 | "consumes": [ 28 | "application/json" 29 | ] 30 | } 31 | }, 32 | "/repos/{remote}/branches": { 33 | "get": { 34 | "summary": "List branches", 35 | "operationId": "Branches_List", 36 | "parameters": [ 37 | { 38 | "type": "string", 39 | "name": "remote", 40 | "required": true, 41 | "in": "path" 42 | }, 43 | { 44 | "name": "x-authorization", 45 | "required": false, 46 | "in": "header", 47 | "type": "string" 48 | }, 49 | { 50 | "name": "x-github-token", 51 | "required": false, 52 | "in": "header", 53 | "type": "string" 54 | } 55 | ], 56 | "responses": { 57 | "200": { 58 | "description": "", 59 | "schema": { 60 | "type": "array", 61 | "items": { 62 | "$ref": "#/definitions/GitBranch" 63 | } 64 | } 65 | }, 66 | "400": { 67 | "description": "When the x-authorization header is malformed" 68 | }, 69 | "404": { 70 | "description": "" 71 | } 72 | }, 73 | "produces": [ 74 | "application/json" 75 | ], 76 | "consumes": [ 77 | "application/json" 78 | ] 79 | } 80 | }, 81 | "/repos/{remote}/commits": { 82 | "get": { 83 | "summary": "List commits", 84 | "operationId": "commits_list", 85 | "parameters": [ 86 | { 87 | "type": "string", 88 | "name": "remote", 89 | "required": true, 90 | "in": "path" 91 | }, 92 | { 93 | "name": "page", 94 | "required": false, 95 | "in": "query", 96 | "type": "string" 97 | }, 98 | { 99 | "name": "ref", 100 | "required": false, 101 | "in": "query", 102 | "description": "Reference to list the commits from. Can be a branch or a commit. Default to master", 103 | "type": "string" 104 | }, 105 | { 106 | "name": "x-authorization", 107 | "required": false, 108 | "in": "header", 109 | "type": "string" 110 | }, 111 | { 112 | "name": "x-github-token", 113 | "required": false, 114 | "in": "header", 115 | "type": "string" 116 | } 117 | ], 118 | "responses": { 119 | "200": { 120 | "headers": { 121 | "Link": { 122 | "type": "string", 123 | "description": "Links to navigate pagination in the format defined by [RFC 5988](https://tools.ietf.org/html/rfc5988#section-5). It will include next, last, first and prev links if applicable" 124 | }, 125 | "x-total-count": { 126 | "type": "integer", 127 | "description": "Total count of items that can be retrieved" 128 | } 129 | }, 130 | "description": "", 131 | "schema": { 132 | "type": "array", 133 | "items": { 134 | "$ref": "#/definitions/GitCommit" 135 | } 136 | } 137 | }, 138 | "400": { 139 | "description": "When the x-authorization header is malformed" 140 | }, 141 | "404": { 142 | "description": "" 143 | } 144 | }, 145 | "produces": [ 146 | "application/json" 147 | ], 148 | "consumes": [ 149 | "application/json" 150 | ] 151 | } 152 | }, 153 | "/repos/{remote}/commits/{commitSha}": { 154 | "get": { 155 | "summary": "Get a commit", 156 | "operationId": "commits_get", 157 | "parameters": [ 158 | { 159 | "type": "string", 160 | "name": "remote", 161 | "required": true, 162 | "in": "path" 163 | }, 164 | { 165 | "type": "string", 166 | "name": "commitSha", 167 | "required": true, 168 | "in": "path" 169 | }, 170 | { 171 | "name": "x-authorization", 172 | "required": false, 173 | "in": "header", 174 | "type": "string" 175 | }, 176 | { 177 | "name": "x-github-token", 178 | "required": false, 179 | "in": "header", 180 | "type": "string" 181 | } 182 | ], 183 | "responses": { 184 | "200": { 185 | "description": "", 186 | "schema": { 187 | "$ref": "#/definitions/GitCommit" 188 | } 189 | }, 190 | "400": { 191 | "description": "When the x-authorization header is malformed" 192 | }, 193 | "404": { 194 | "description": "" 195 | } 196 | }, 197 | "produces": [ 198 | "application/json" 199 | ], 200 | "consumes": [ 201 | "application/json" 202 | ] 203 | } 204 | }, 205 | "/repos/{remote}/compare/{base}...{head}": { 206 | "get": { 207 | "summary": "Compare two commits", 208 | "operationId": "commits_compare", 209 | "parameters": [ 210 | { 211 | "type": "string", 212 | "name": "remote", 213 | "required": true, 214 | "in": "path" 215 | }, 216 | { 217 | "type": "string", 218 | "name": "base", 219 | "required": true, 220 | "in": "path" 221 | }, 222 | { 223 | "type": "string", 224 | "name": "head", 225 | "required": true, 226 | "in": "path" 227 | }, 228 | { 229 | "name": "x-authorization", 230 | "required": false, 231 | "in": "header", 232 | "type": "string" 233 | }, 234 | { 235 | "name": "x-github-token", 236 | "required": false, 237 | "in": "header", 238 | "type": "string" 239 | } 240 | ], 241 | "responses": { 242 | "200": { 243 | "description": "", 244 | "schema": { 245 | "$ref": "#/definitions/GitDiff" 246 | } 247 | }, 248 | "400": { 249 | "description": "When the x-authorization header is malformed" 250 | }, 251 | "404": { 252 | "description": "" 253 | } 254 | }, 255 | "produces": [ 256 | "application/json" 257 | ], 258 | "consumes": [ 259 | "application/json" 260 | ] 261 | } 262 | }, 263 | "/repos/{remote}/contents/{path}": { 264 | "get": { 265 | "summary": "Get content", 266 | "operationId": "contents_get", 267 | "parameters": [ 268 | { 269 | "type": "string", 270 | "name": "remote", 271 | "required": true, 272 | "in": "path" 273 | }, 274 | { 275 | "name": "path", 276 | "required": true, 277 | "in": "path", 278 | "type": "string" 279 | }, 280 | { 281 | "name": "recursive", 282 | "required": false, 283 | "in": "query", 284 | "type": "string" 285 | }, 286 | { 287 | "name": "ref", 288 | "required": false, 289 | "in": "query", 290 | "type": "string" 291 | }, 292 | { 293 | "name": "x-authorization", 294 | "required": false, 295 | "in": "header", 296 | "type": "string" 297 | }, 298 | { 299 | "name": "x-github-token", 300 | "required": false, 301 | "in": "header", 302 | "type": "string" 303 | } 304 | ], 305 | "responses": { 306 | "200": { 307 | "description": "", 308 | "schema": { 309 | "$ref": "#/definitions/GitContents" 310 | } 311 | }, 312 | "400": { 313 | "description": "When the x-authorization header is malformed" 314 | }, 315 | "404": { 316 | "description": "" 317 | } 318 | }, 319 | "produces": [ 320 | "application/json" 321 | ], 322 | "consumes": [ 323 | "application/json" 324 | ] 325 | } 326 | }, 327 | "/repos/{remote}/tree/{path}": { 328 | "get": { 329 | "summary": "Get tree", 330 | "operationId": "tree_get", 331 | "parameters": [ 332 | { 333 | "type": "string", 334 | "name": "remote", 335 | "required": true, 336 | "in": "path" 337 | }, 338 | { 339 | "name": "path", 340 | "required": true, 341 | "in": "path", 342 | "type": "string" 343 | }, 344 | { 345 | "name": "ref", 346 | "required": false, 347 | "in": "query", 348 | "type": "string" 349 | }, 350 | { 351 | "name": "x-authorization", 352 | "required": false, 353 | "in": "header", 354 | "type": "string" 355 | }, 356 | { 357 | "name": "x-github-token", 358 | "required": false, 359 | "in": "header", 360 | "type": "string" 361 | } 362 | ], 363 | "responses": { 364 | "200": { 365 | "description": "", 366 | "schema": { 367 | "$ref": "#/definitions/GitTree" 368 | } 369 | }, 370 | "400": { 371 | "description": "When the x-authorization header is malformed" 372 | }, 373 | "404": { 374 | "description": "" 375 | } 376 | }, 377 | "produces": [ 378 | "application/json" 379 | ], 380 | "consumes": [ 381 | "application/json" 382 | ] 383 | } 384 | } 385 | }, 386 | "definitions": { 387 | "GitCommitRef": { 388 | "type": "object", 389 | "properties": { 390 | "sha": { 391 | "type": "string" 392 | } 393 | }, 394 | "required": [ 395 | "sha" 396 | ] 397 | }, 398 | "GitBranch": { 399 | "type": "object", 400 | "properties": { 401 | "name": { 402 | "type": "string" 403 | }, 404 | "commit": { 405 | "$ref": "#/definitions/GitCommitRef" 406 | } 407 | }, 408 | "required": [ 409 | "name", 410 | "commit" 411 | ] 412 | }, 413 | "GitSignature": { 414 | "type": "object", 415 | "properties": { 416 | "name": { 417 | "type": "string" 418 | }, 419 | "email": { 420 | "type": "string" 421 | }, 422 | "date": { 423 | "type": "string", 424 | "format": "date-time" 425 | } 426 | }, 427 | "required": [ 428 | "name", 429 | "email", 430 | "date" 431 | ] 432 | }, 433 | "GitCommit": { 434 | "type": "object", 435 | "properties": { 436 | "sha": { 437 | "type": "string" 438 | }, 439 | "message": { 440 | "type": "string" 441 | }, 442 | "author": { 443 | "$ref": "#/definitions/GitSignature" 444 | }, 445 | "committer": { 446 | "$ref": "#/definitions/GitSignature" 447 | }, 448 | "parents": { 449 | "type": "array", 450 | "items": { 451 | "$ref": "#/definitions/GitCommitRef" 452 | } 453 | } 454 | }, 455 | "required": [ 456 | "sha", 457 | "message", 458 | "author", 459 | "committer", 460 | "parents" 461 | ] 462 | }, 463 | "GitFileDiff": { 464 | "type": "object", 465 | "properties": { 466 | "filename": { 467 | "type": "string" 468 | }, 469 | "sha": { 470 | "type": "string" 471 | }, 472 | "status": { 473 | "type": "string", 474 | "enum": [ 475 | "unmodified", 476 | "modified", 477 | "added", 478 | "deleted", 479 | "renamed" 480 | ], 481 | "x-ms-enum": { 482 | "name": "PatchStatus" 483 | } 484 | }, 485 | "additions": { 486 | "type": "number" 487 | }, 488 | "deletions": { 489 | "type": "number" 490 | }, 491 | "changes": { 492 | "type": "number" 493 | }, 494 | "previousFilename": { 495 | "type": "string" 496 | } 497 | }, 498 | "required": [ 499 | "filename", 500 | "sha", 501 | "status", 502 | "additions", 503 | "deletions", 504 | "changes" 505 | ] 506 | }, 507 | "GitDiff": { 508 | "type": "object", 509 | "properties": { 510 | "headCommit": { 511 | "$ref": "#/definitions/GitCommit" 512 | }, 513 | "baseCommit": { 514 | "$ref": "#/definitions/GitCommit" 515 | }, 516 | "mergeBaseCommit": { 517 | "$ref": "#/definitions/GitCommit" 518 | }, 519 | "totalCommits": { 520 | "type": "number" 521 | }, 522 | "commits": { 523 | "type": "array", 524 | "items": { 525 | "$ref": "#/definitions/GitCommit" 526 | } 527 | }, 528 | "files": { 529 | "type": "array", 530 | "items": { 531 | "$ref": "#/definitions/GitFileDiff" 532 | } 533 | } 534 | }, 535 | "required": [ 536 | "headCommit", 537 | "baseCommit", 538 | "mergeBaseCommit", 539 | "totalCommits", 540 | "commits", 541 | "files" 542 | ] 543 | }, 544 | "GitDirObjectContent": { 545 | "type": "object", 546 | "properties": { 547 | "type": { 548 | "type": "string" 549 | }, 550 | "size": { 551 | "type": "number" 552 | }, 553 | "name": { 554 | "type": "string" 555 | }, 556 | "path": { 557 | "type": "string" 558 | }, 559 | "sha": { 560 | "type": "string" 561 | } 562 | }, 563 | "required": [ 564 | "type", 565 | "size", 566 | "name", 567 | "path", 568 | "sha" 569 | ] 570 | }, 571 | "GitFileObjectWithoutContent": { 572 | "type": "object", 573 | "properties": { 574 | "type": { 575 | "type": "string" 576 | }, 577 | "size": { 578 | "type": "number" 579 | }, 580 | "name": { 581 | "type": "string" 582 | }, 583 | "path": { 584 | "type": "string" 585 | }, 586 | "sha": { 587 | "type": "string" 588 | }, 589 | "encoding": { 590 | "type": "string" 591 | } 592 | }, 593 | "required": [ 594 | "type", 595 | "size", 596 | "name", 597 | "path", 598 | "sha", 599 | "encoding" 600 | ] 601 | }, 602 | "GitSubmoduleObjectContent": { 603 | "type": "object", 604 | "properties": { 605 | "type": { 606 | "type": "string" 607 | }, 608 | "size": { 609 | "type": "number" 610 | }, 611 | "name": { 612 | "type": "string" 613 | }, 614 | "path": { 615 | "type": "string" 616 | }, 617 | "sha": { 618 | "type": "string" 619 | } 620 | }, 621 | "required": [ 622 | "type", 623 | "size", 624 | "name", 625 | "path", 626 | "sha" 627 | ] 628 | }, 629 | "GitContents": { 630 | "type": "object", 631 | "properties": { 632 | "dirs": { 633 | "type": "array", 634 | "items": { 635 | "$ref": "#/definitions/GitDirObjectContent" 636 | } 637 | }, 638 | "files": { 639 | "type": "array", 640 | "items": { 641 | "$ref": "#/definitions/GitFileObjectWithoutContent" 642 | } 643 | }, 644 | "submodules": { 645 | "type": "array", 646 | "items": { 647 | "$ref": "#/definitions/GitSubmoduleObjectContent" 648 | } 649 | } 650 | }, 651 | "required": [ 652 | "dirs", 653 | "files", 654 | "submodules" 655 | ] 656 | }, 657 | "GitFileObjectWithContent": { 658 | "type": "object", 659 | "properties": { 660 | "type": { 661 | "type": "string" 662 | }, 663 | "size": { 664 | "type": "number" 665 | }, 666 | "name": { 667 | "type": "string" 668 | }, 669 | "path": { 670 | "type": "string" 671 | }, 672 | "sha": { 673 | "type": "string" 674 | }, 675 | "encoding": { 676 | "type": "string" 677 | }, 678 | "content": { 679 | "type": "string" 680 | } 681 | }, 682 | "required": [ 683 | "type", 684 | "size", 685 | "name", 686 | "path", 687 | "sha", 688 | "encoding", 689 | "content" 690 | ] 691 | }, 692 | "GitTree": { 693 | "type": "object", 694 | "properties": { 695 | "dirs": { 696 | "type": "array", 697 | "items": { 698 | "$ref": "#/definitions/GitDirObjectContent" 699 | } 700 | }, 701 | "files": { 702 | "type": "array", 703 | "items": { 704 | "$ref": "#/definitions/GitFileObjectWithContent" 705 | } 706 | }, 707 | "submodules": { 708 | "type": "array", 709 | "items": { 710 | "$ref": "#/definitions/GitSubmoduleObjectContent" 711 | } 712 | } 713 | }, 714 | "required": [ 715 | "dirs", 716 | "files", 717 | "submodules" 718 | ] 719 | } 720 | } 721 | } --------------------------------------------------------------------------------