├── src ├── types │ └── global.d.ts ├── integration_tests │ ├── dummy-gcp-credentials.json │ ├── docker-compose.yml │ ├── run.sh │ └── main.test.ts ├── lib │ ├── errors.ts │ ├── StoreObject.ts │ ├── config.ts │ ├── adapters │ │ ├── index.ts │ │ ├── MinioClient.ts │ │ ├── _test_utils.ts │ │ ├── MinioClient.spec.ts │ │ ├── GCSClient.ts │ │ ├── S3Client.ts │ │ ├── GCSClient.spec.ts │ │ └── S3Client.spec.ts │ ├── ObjectStoreClient.ts │ ├── init.ts │ ├── _test_utils.ts │ └── init.spec.ts └── index.ts ├── .gitignore ├── .releaserc.yml ├── tslint.json ├── .prettierignore ├── .env ├── typedoc.json ├── .npmignore ├── tsconfig.module.json ├── .github ├── workflows │ ├── prs.yml │ └── ci.yml └── dependabot.yml ├── jest.config.ci.js ├── tsconfig.json ├── jest.config.integration.js ├── .editorconfig ├── LICENSE ├── README.md ├── package.json └── jest.config.js /src/types/global.d.ts: -------------------------------------------------------------------------------- 1 | import 'jest-extended'; 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | build 3 | src/**.js 4 | coverage 5 | -------------------------------------------------------------------------------- /.releaserc.yml: -------------------------------------------------------------------------------- 1 | branches: [main] 2 | extends: '@relaycorp/shared-config' 3 | -------------------------------------------------------------------------------- /tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["@relaycorp/shared-config/tslint.json"] 3 | } 4 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | # package.json is formatted by package managers, so we ignore it here 2 | package.json -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | OBJECT_STORE_BUCKET=the-bucket 2 | OBJECT_STORE_ACCESS_KEY_ID=THE-KEY-ID 3 | OBJECT_STORE_SECRET_KEY=letmeinpls 4 | -------------------------------------------------------------------------------- /typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "exclude": ["**/*.spec.ts"], 3 | "excludePrivate": true, 4 | "excludeProtected": true, 5 | "theme": "default" 6 | } 7 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | * 2 | 3 | !build/main/** 4 | !build/module/** 5 | _test_utils.* 6 | *.spec.js* 7 | 8 | !LICENSE 9 | !README.md 10 | !package.json 11 | -------------------------------------------------------------------------------- /src/integration_tests/dummy-gcp-credentials.json: -------------------------------------------------------------------------------- 1 | { 2 | "client_email": "foo@example.com", 3 | "private_key": "private", 4 | "project_id": "dummy-project" 5 | } 6 | -------------------------------------------------------------------------------- /src/lib/errors.ts: -------------------------------------------------------------------------------- 1 | import VError from 'verror'; 2 | 3 | export class ObjectStorageError extends VError { 4 | public override readonly name = 'ObjectStorageError'; 5 | } 6 | -------------------------------------------------------------------------------- /src/lib/StoreObject.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Store object. 3 | */ 4 | export interface StoreObject { 5 | readonly metadata: { readonly [key: string]: string }; 6 | readonly body: Buffer; 7 | } 8 | -------------------------------------------------------------------------------- /tsconfig.module.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig", 3 | "compilerOptions": { 4 | "target": "esnext", 5 | "outDir": "build/module", 6 | "module": "esnext" 7 | }, 8 | "exclude": [ 9 | "node_modules/**" 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /.github/workflows/prs.yml: -------------------------------------------------------------------------------- 1 | name: Process PRs 2 | 3 | on: 4 | pull_request: 5 | types: 6 | - opened 7 | - edited 8 | - synchronize 9 | 10 | jobs: 11 | pr-ci: 12 | uses: relaycorp/shared-workflows/.github/workflows/pr-ci.yml@main 13 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export { StoreObject } from './lib/StoreObject'; 2 | export * from './lib/errors'; 3 | export { ObjectStoreClient } from './lib/ObjectStoreClient'; 4 | export { initObjectStoreClient } from './lib/init'; 5 | export { AdapterType } from './lib/adapters'; 6 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: npm 4 | directory: "/" 5 | schedule: 6 | interval: weekly 7 | versioning-strategy: increase 8 | commit-message: 9 | prefix: "fix(deps):" 10 | prefix-development: "chore(deps):" 11 | -------------------------------------------------------------------------------- /jest.config.ci.js: -------------------------------------------------------------------------------- 1 | const mainJestConfig = require('./jest.config'); 2 | 3 | module.exports = Object.assign({}, mainJestConfig, { 4 | collectCoverageFrom: ["build/main/lib/**/*.js"], 5 | moduleFileExtensions: ['js'], 6 | preset: null, 7 | roots: ['build/main/lib'] 8 | }); 9 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@relaycorp/shared-config", 3 | "compilerOptions": { 4 | "outDir": "build/main", 5 | "rootDir": "src", 6 | "types": ["node", "jest"], 7 | "typeRoots": ["node_modules/@types"] 8 | }, 9 | "include": ["src/**/*.ts"] 10 | } 11 | -------------------------------------------------------------------------------- /jest.config.integration.js: -------------------------------------------------------------------------------- 1 | const mainJestConfig = require('./jest.config'); 2 | 3 | module.exports = { 4 | preset: mainJestConfig.preset, 5 | roots: ['build/main/integration_tests'], 6 | testEnvironment: mainJestConfig.testEnvironment, 7 | setupFilesAfterEnv: mainJestConfig.setupFilesAfterEnv 8 | }; 9 | -------------------------------------------------------------------------------- /src/lib/config.ts: -------------------------------------------------------------------------------- 1 | export interface ClientConfig { 2 | readonly credentials?: ClientCredentials; 3 | readonly endpoint?: string; 4 | readonly tlsEnabled: boolean; 5 | } 6 | 7 | export interface ClientCredentials { 8 | readonly accessKeyId: string; 9 | readonly secretAccessKey: string; 10 | } 11 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | charset = utf-8 6 | end_of_line = lf 7 | indent_size = 2 8 | indent_style = space 9 | insert_final_newline = true 10 | max_line_length = 100 11 | trim_trailing_whitespace = true 12 | 13 | [*.md] 14 | max_line_length = off 15 | trim_trailing_whitespace = false 16 | -------------------------------------------------------------------------------- /src/lib/adapters/index.ts: -------------------------------------------------------------------------------- 1 | import { GCSClient } from './GCSClient'; 2 | import { MinioClient } from './MinioClient'; 3 | import { S3Client } from './S3Client'; 4 | 5 | export type AdapterType = 'gcs' | 'minio' | 's3'; 6 | 7 | export const CLIENT_BY_ADAPTER_NAME = { 8 | gcs: GCSClient, 9 | minio: MinioClient, 10 | s3: S3Client, 11 | }; 12 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI and automated releases 2 | on: 3 | pull_request: 4 | push: 5 | branches: [ main ] 6 | jobs: 7 | ci: 8 | uses: relaycorp/shared-workflows/.github/workflows/nodejs-lib-ci.yml@main 9 | 10 | release: 11 | needs: ci 12 | uses: relaycorp/shared-workflows/.github/workflows/nodejs-lib-release.yml@main 13 | with: 14 | api_docs: true 15 | secrets: 16 | npm_token: ${{ secrets.NPM_TOKEN }} 17 | -------------------------------------------------------------------------------- /src/lib/adapters/MinioClient.ts: -------------------------------------------------------------------------------- 1 | import { ClientConfig } from '../config'; 2 | import { ObjectStorageError } from '../errors'; 3 | import { S3Client } from './S3Client'; 4 | 5 | export class MinioClient extends S3Client { 6 | constructor(config: ClientConfig) { 7 | if (!config.endpoint) { 8 | throw new ObjectStorageError('The Minio adapter requires an endpoint'); 9 | } 10 | if (!config.credentials) { 11 | throw new ObjectStorageError('The Minio adapter requires credentials'); 12 | } 13 | super(config); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/lib/adapters/_test_utils.ts: -------------------------------------------------------------------------------- 1 | import { ENDPOINT } from '../_test_utils'; 2 | import { ClientConfig } from '../config'; 3 | import { StoreObject } from '../StoreObject'; 4 | 5 | export const BUCKET = 'the-bucket'; 6 | export const OBJECT: StoreObject = { body: Buffer.from('the-body'), metadata: { key: 'value' } }; 7 | export const OBJECT_PREFIX = 'prefix/'; 8 | export const OBJECT1_KEY = `${OBJECT_PREFIX}the-object.txt`; 9 | export const OBJECT2_KEY = `${OBJECT_PREFIX}another-object.txt`; 10 | 11 | export const CLIENT_CONFIG: ClientConfig = { 12 | endpoint: ENDPOINT, 13 | tlsEnabled: true, 14 | }; 15 | -------------------------------------------------------------------------------- /src/integration_tests/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | 3 | services: 4 | minio: 5 | image: minio/minio:RELEASE.2021-01-16T02-19-44Z 6 | init: true 7 | command: ["server", "/data"] 8 | environment: 9 | MINIO_ACCESS_KEY: ${OBJECT_STORE_ACCESS_KEY_ID} 10 | MINIO_SECRET_KEY: ${OBJECT_STORE_SECRET_KEY} 11 | MINIO_API_READY_DEADLINE: 5s 12 | healthcheck: 13 | test: ["CMD", "curl", "--fail", "http://localhost:9000/minio/health/ready"] 14 | ports: 15 | - '127.0.0.1:9000:9000' 16 | 17 | gcs: 18 | image: fsouza/fake-gcs-server:v1.29.0 19 | init: true 20 | command: ["-scheme", "http", "-port", "8080"] 21 | ports: 22 | - '127.0.0.1:8080:8080' 23 | -------------------------------------------------------------------------------- /src/integration_tests/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | 3 | set -o nounset 4 | set -o errexit 5 | set -o pipefail 6 | 7 | CURRENT_DIR="$(dirname "${BASH_SOURCE[0]}")" 8 | 9 | export COMPOSE_PROJECT_NAME='object-storage-functional-tests' 10 | export COMPOSE_FILE="${CURRENT_DIR}/docker-compose.yml" 11 | 12 | trap "docker-compose down --remove-orphans --volumes" INT TERM EXIT 13 | 14 | docker-compose pull 15 | 16 | docker-compose --env-file "$(pwd)/.env" up --force-recreate --detach 17 | sleep 3s 18 | 19 | docker-compose ps 20 | 21 | export GOOGLE_APPLICATION_CREDENTIALS="${CURRENT_DIR}/dummy-gcp-credentials.json" 22 | if ! jest --config jest.config.integration.js --runInBand --detectOpenHandles ; then 23 | docker-compose logs gcs 24 | exit 1 25 | fi 26 | -------------------------------------------------------------------------------- /src/lib/adapters/MinioClient.spec.ts: -------------------------------------------------------------------------------- 1 | import { CLIENT_CREDENTIALS, ENDPOINT } from '../_test_utils'; 2 | import { ObjectStorageError } from '../errors'; 3 | import { MinioClient } from './MinioClient'; 4 | 5 | jest.mock('aws-sdk'); 6 | 7 | describe('Constructor', () => { 8 | test('Error should be thrown if endpoint is not set', () => { 9 | // tslint:disable-next-line:no-unused-expression 10 | expect(() => new MinioClient({ credentials: CLIENT_CREDENTIALS, tlsEnabled: true })).toThrow( 11 | new ObjectStorageError('The Minio adapter requires an endpoint'), 12 | ); 13 | }); 14 | 15 | test('Error should be thrown if credentials are not set', () => { 16 | // tslint:disable-next-line:no-unused-expression 17 | expect(() => new MinioClient({ endpoint: ENDPOINT, tlsEnabled: true })).toThrow( 18 | new ObjectStorageError('The Minio adapter requires credentials'), 19 | ); 20 | }); 21 | }); 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021-2022 Relaycorp, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/lib/ObjectStoreClient.ts: -------------------------------------------------------------------------------- 1 | import { StoreObject } from './StoreObject'; 2 | 3 | /** 4 | * Interface for all object store clients. 5 | */ 6 | export interface ObjectStoreClient { 7 | /** 8 | * Retrieve the object by key or return `null` if the key doesn't exist. 9 | */ 10 | readonly getObject: (key: string, bucket: string) => Promise; 11 | 12 | /** 13 | * Retrieve the keys for all the objects in the specified bucket with the specified key prefix. 14 | * 15 | * This will handle pagination if the backend supports it (S3 does, Minio doesn't). 16 | * 17 | * @param prefix 18 | * @param bucket 19 | */ 20 | readonly listObjectKeys: (prefix: string, bucket: string) => AsyncIterable; 21 | 22 | readonly putObject: (object: StoreObject, key: string, bucket: string) => Promise; 23 | 24 | /** 25 | * Delete the object corresponding to `key`. 26 | * 27 | * This method resolves even if the `key` doesn't exist, in order to provide a normalised API, 28 | * since Amazon S3 always returns a `204` regardless of whether the object exists (GCS returns a 29 | * `404` and its SDK throws an error). 30 | */ 31 | readonly deleteObject: (key: string, bucket: string) => Promise; 32 | } 33 | -------------------------------------------------------------------------------- /src/lib/init.ts: -------------------------------------------------------------------------------- 1 | import { AdapterType, CLIENT_BY_ADAPTER_NAME } from './adapters'; 2 | import { ClientConfig } from './config'; 3 | import { ObjectStorageError } from './errors'; 4 | import { ObjectStoreClient } from './ObjectStoreClient'; 5 | 6 | type clientConstructor = (config: ClientConfig) => ObjectStoreClient; 7 | 8 | const CONSTRUCTORS: { readonly [key: string]: clientConstructor } = Object.entries( 9 | CLIENT_BY_ADAPTER_NAME, 10 | ).reduce((acc, [k, v]) => ({ ...acc, [k]: (config: ClientConfig) => new v(config) }), {}); 11 | 12 | /** 13 | * Return object store client for the specified `type`. 14 | * 15 | * @param type 16 | * @param endpoint 17 | * @param accessKeyId 18 | * @param secretAccessKey 19 | * @param tlsEnabled 20 | */ 21 | export function initObjectStoreClient( 22 | type: AdapterType, 23 | endpoint?: string, 24 | accessKeyId?: string, 25 | secretAccessKey?: string, 26 | tlsEnabled = true, 27 | ): ObjectStoreClient { 28 | const constructor = CONSTRUCTORS[type]; 29 | if (!constructor) { 30 | throw new ObjectStorageError(`Unknown client type "${type}"`); 31 | } 32 | if ((accessKeyId && !secretAccessKey) || (!accessKeyId && secretAccessKey)) { 33 | throw new ObjectStorageError( 34 | 'Both access key id and secret access key must be set, or neither must be set', 35 | ); 36 | } 37 | const credentials = accessKeyId && secretAccessKey ? { accessKeyId, secretAccessKey } : undefined; 38 | return constructor({ 39 | credentials, 40 | endpoint, 41 | tlsEnabled, 42 | }); 43 | } 44 | -------------------------------------------------------------------------------- /src/lib/_test_utils.ts: -------------------------------------------------------------------------------- 1 | import { ClientCredentials } from './config'; 2 | 3 | export const SECRET_ACCESS_KEY = 'secret-access-key'; 4 | export const ACCESS_KEY = 'the-access-key'; 5 | export const ENDPOINT = 'the-endpoint:1234'; 6 | export const CLIENT_CREDENTIALS: ClientCredentials = { 7 | accessKeyId: ACCESS_KEY, 8 | secretAccessKey: SECRET_ACCESS_KEY, 9 | }; 10 | 11 | export async function asyncIterableToArray(iterable: AsyncIterable): Promise { 12 | // tslint:disable-next-line:readonly-array 13 | const values = []; 14 | for await (const item of iterable) { 15 | values.push(item); 16 | } 17 | return values; 18 | } 19 | 20 | // tslint:disable-next-line:readonly-array 21 | export function mockSpy( 22 | spy: jest.MockInstance, 23 | mockImplementation?: (...args: readonly any[]) => any, 24 | ): jest.MockInstance { 25 | beforeEach(() => { 26 | spy.mockReset(); 27 | if (mockImplementation) { 28 | spy.mockImplementation(mockImplementation); 29 | } 30 | }); 31 | 32 | afterAll(() => { 33 | spy.mockRestore(); 34 | }); 35 | 36 | return spy; 37 | } 38 | 39 | function getMockInstance(mockedObject: any): jest.MockInstance { 40 | return mockedObject as unknown as jest.MockInstance; 41 | } 42 | 43 | export function getMockContext(mockedObject: any): jest.MockContext { 44 | const mockInstance = getMockInstance(mockedObject); 45 | return mockInstance.mock; 46 | } 47 | 48 | export async function getPromiseRejection( 49 | promise: Promise, 50 | expectedErrorType: new () => ErrorType, 51 | ): Promise { 52 | try { 53 | await promise; 54 | } catch (error) { 55 | expect(error).toBeInstanceOf(expectedErrorType); 56 | return error as ErrorType; 57 | } 58 | throw new Error('Expected promise to throw'); 59 | } 60 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # @relaycorp/object-storage 2 | 3 | This library allows you to interact with major object storage backends (e.g., Amazon S3) using a unified interface. 4 | 5 | ## Install 6 | 7 | Releases are automatically pushed to [NPM](https://www.npmjs.com/package/@relaycorp/object-storage). For example, to get the latest release, run: 8 | 9 | ``` 10 | npm install @relaycorp/object-storage 11 | ``` 12 | 13 | ## Supported backends 14 | 15 | | Backend | Name | SDK used internally | Access/secret key support | 16 | | --- | --- | --- | --- | 17 | | Amazon S3 | `s3` | AWS | Optional | 18 | | Google Cloud Storage (GCS) | `gcs` | GCP | Unsupported. Use alternative authentication methods instead, like [workload identity](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity). | 19 | | Minio | `minio` | AWS | Required | 20 | 21 | ## Supported operations 22 | 23 | - Get object (along with its metadata). 24 | - Create/update object (along with its metadata). 25 | - Delete object. 26 | - List objects under a given prefix. Pagination is handled automatically: We return an iterable and retrieve the next page lazily. 27 | 28 | We welcome PRs to support additional operations, as long as each new operation is supported across all backends in a normalised way, and has unit and functional tests. 29 | 30 | ## Example 31 | 32 | ```typescript 33 | import { initObjectStoreClient } from '@relaycorp/object-storage'; 34 | 35 | async function main(): Promise { 36 | const client = initObjectStoreClient( 37 | process.env.STORE_BACKEND, 38 | process.env.STORE_ENDPOINT, 39 | process.env.STORE_ACCESS_KEY, 40 | process.env.STORE_SECRET_KEY, 41 | ); 42 | 43 | for await (const objectKey of client.listObjectKeys("prefix/", process.env.STORE_BUCKET)) { 44 | console.log("- ", objectKey) 45 | } 46 | } 47 | ``` 48 | 49 | ## API documentation 50 | 51 | The API documentation can be found on [docs.relaycorp.tech](https://docs.relaycorp.tech/object-storage-js/). 52 | 53 | ## Contributing 54 | 55 | We love contributions! If you haven't contributed to a Relaycorp project before, please take a minute to [read our guidelines](https://github.com/relaycorp/.github/blob/master/CONTRIBUTING.md) first. 56 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@relaycorp/object-storage", 3 | "version": "1.0.0", 4 | "author": { 5 | "email": "no-reply@relaycorp.tech", 6 | "name": "Relaycorp, Inc.", 7 | "url": "https://relaycorp.tech/" 8 | }, 9 | "description": "Node.js client for object stores like AWS S3, GCP GCS or Minio", 10 | "main": "build/main/index.js", 11 | "typings": "build/main/index.d.ts", 12 | "module": "build/module/index.js", 13 | "repository": "https://github.com/relaycorp/object-storage-js", 14 | "homepage": "https://docs.relaycorp.tech/object-storage-js/", 15 | "license": "MIT", 16 | "keywords": [ 17 | "gcs", 18 | "minio", 19 | "s3" 20 | ], 21 | "scripts": { 22 | "build": "run-s clean && run-p build:*", 23 | "build:main": "tsc -p tsconfig.json", 24 | "build:module": "tsc -p tsconfig.module.json", 25 | "fix": "run-s fix:*", 26 | "fix:prettier": "prettier \"src/**/*.ts\" --write", 27 | "fix:tslint": "tslint --fix --project .", 28 | "static-checks": "run-p static-checks:*", 29 | "static-checks:lint": "tslint --project .", 30 | "static-checks:prettier": "prettier \"src/**/*.ts\" --list-different", 31 | "test": "jest --coverage", 32 | "test:ci:unit": "run-s build test:ci:unit:jest", 33 | "test:ci:unit:jest": "jest --config jest.config.ci.js --coverage", 34 | "pretest:integration": "run-s build", 35 | "test:integration": "src/integration_tests/run.sh", 36 | "doc-api": "typedoc src/index.ts --out build/docs", 37 | "clean": "del-cli build test" 38 | }, 39 | "engines": { 40 | "node": ">=14" 41 | }, 42 | "dependencies": { 43 | "@google-cloud/storage": "^7.9.0", 44 | "aws-sdk": "^2.1637.0", 45 | "verror": "^1.10.1" 46 | }, 47 | "devDependencies": { 48 | "@relaycorp/shared-config": "^1.14.1", 49 | "@types/jest": "^29.5.12", 50 | "@types/minio": "^7.1.1", 51 | "@types/verror": "^1.10.10", 52 | "del-cli": "^5.1.0", 53 | "dotenv": "^16.0.3", 54 | "env-var": "^7.5.0", 55 | "gh-pages": "^6.1.1", 56 | "jest": "^29.7.0", 57 | "jest-date-mock": "^1.0.8", 58 | "jest-extended": "^4.0.2", 59 | "minio": "^7.1.4", 60 | "npm-run-all": "^4.1.5", 61 | "prettier": "^2.8.8", 62 | "ts-jest": "^29.1.4", 63 | "ts-node": "^10.9.2", 64 | "tslint": "^6.1.3", 65 | "typedoc": "^0.25.13", 66 | "typescript": "^5.4.5" 67 | }, 68 | "prettier": "@relaycorp/shared-config/.prettierrc.json", 69 | "publishConfig": { 70 | "access": "public", 71 | "tag": "latest" 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/lib/adapters/GCSClient.ts: -------------------------------------------------------------------------------- 1 | import { File, GetFilesOptions, Storage } from '@google-cloud/storage'; 2 | 3 | import { ClientConfig } from '../config'; 4 | import { ObjectStorageError } from '../errors'; 5 | import { ObjectStoreClient } from '../ObjectStoreClient'; 6 | import { StoreObject } from '../StoreObject'; 7 | 8 | export class GCSClient implements ObjectStoreClient { 9 | protected readonly client: Storage; 10 | 11 | constructor(config: ClientConfig) { 12 | if (config.credentials) { 13 | throw new ObjectStorageError('GCS adapter does not accept HMAC key credentials'); 14 | } 15 | if (!config.endpoint && !config.tlsEnabled) { 16 | throw new ObjectStorageError('GCS adapter requires custom endpoint if TLS is to be disabled'); 17 | } 18 | const apiEndpointScheme = config.tlsEnabled ? 'https' : 'http'; 19 | const apiEndpoint = config.endpoint ? `${apiEndpointScheme}://${config.endpoint}` : undefined; 20 | this.client = new Storage({ ...(apiEndpoint && { apiEndpoint }) }); 21 | } 22 | 23 | public async deleteObject(key: string, bucket: string): Promise { 24 | const file = this.client.bucket(bucket).file(key); 25 | try { 26 | await file.delete(); 27 | } catch (err) { 28 | if ((err as any).code !== 404) { 29 | throw new ObjectStorageError(err as Error, 'Failed to delete object'); 30 | } 31 | } 32 | } 33 | 34 | public async getObject(key: string, bucket: string): Promise { 35 | let gcsFile: File; 36 | try { 37 | [gcsFile] = await this.client.bucket(bucket).file(key).get(); 38 | } catch (err) { 39 | if ((err as any).code === 404) { 40 | return null; 41 | } 42 | throw new ObjectStorageError(err as Error, 'Failed to retrieve object'); 43 | } 44 | const download = await gcsFile.download(); 45 | return { body: download[0], metadata: gcsFile.metadata.metadata ?? {} }; 46 | } 47 | 48 | public async *listObjectKeys(prefix: string, bucket: string): AsyncIterable { 49 | let query: GetFilesOptions = { prefix, autoPaginate: false }; 50 | do { 51 | const retrievalResult = await this.client.bucket(bucket).getFiles(query); 52 | for (const file of retrievalResult[0]) { 53 | yield file.name; 54 | } 55 | query = retrievalResult[1]; 56 | } while (query !== null); 57 | } 58 | 59 | public async putObject(object: StoreObject, key: string, bucket: string): Promise { 60 | const file = this.client.bucket(bucket).file(key); 61 | await file.save(object.body, { 62 | resumable: false, // https://github.com/fsouza/fake-gcs-server/issues/346 63 | }); 64 | await file.setMetadata({ metadata: object.metadata }); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/lib/adapters/S3Client.ts: -------------------------------------------------------------------------------- 1 | // tslint:disable:no-submodule-imports 2 | import { S3 } from 'aws-sdk'; 3 | import { GetObjectOutput } from 'aws-sdk/clients/s3'; 4 | import * as http from 'http'; 5 | import * as https from 'https'; 6 | 7 | import { ClientConfig } from '../config'; 8 | import { ObjectStoreClient } from '../ObjectStoreClient'; 9 | import { StoreObject } from '../StoreObject'; 10 | 11 | /** 12 | * Thin wrapper around S3-compatible object storage library. 13 | */ 14 | export class S3Client implements ObjectStoreClient { 15 | protected readonly client: S3; 16 | 17 | constructor(config: ClientConfig) { 18 | const agentOptions = { keepAlive: true }; 19 | const agent = config.tlsEnabled ? new https.Agent(agentOptions) : new http.Agent(agentOptions); 20 | const options = { 21 | httpOptions: { agent }, 22 | s3ForcePathStyle: true, 23 | signatureVersion: 'v4', 24 | sslEnabled: config.tlsEnabled, 25 | }; 26 | this.client = new S3({ 27 | ...options, 28 | ...(config.credentials && config.credentials), 29 | ...(config.endpoint && { endpoint: config.endpoint }), 30 | }); 31 | } 32 | 33 | public async getObject(key: string, bucket: string): Promise { 34 | let s3Object: GetObjectOutput; 35 | try { 36 | s3Object = await this.client.getObject({ Bucket: bucket, Key: key }).promise(); 37 | } catch (err) { 38 | if ((err as any).code === 'NoSuchKey') { 39 | return null; 40 | } 41 | throw err; 42 | } 43 | return { body: s3Object.Body as Buffer, metadata: s3Object.Metadata || {} }; 44 | } 45 | 46 | public async *listObjectKeys(prefix: string, bucket: string): AsyncIterable { 47 | let continuationToken: string | undefined; 48 | do { 49 | const request = this.client.listObjectsV2({ 50 | Bucket: bucket, 51 | Prefix: prefix, 52 | ...(continuationToken && { ContinuationToken: continuationToken }), 53 | }); 54 | const response = await request.promise(); 55 | for (const objectData of response.Contents as S3.ObjectList) { 56 | yield objectData.Key as string; 57 | } 58 | 59 | continuationToken = response.NextContinuationToken; 60 | } while (continuationToken !== undefined); 61 | } 62 | 63 | public async putObject(object: StoreObject, key: string, bucket: string): Promise { 64 | const request = this.client.putObject({ 65 | Body: object.body, 66 | Bucket: bucket, 67 | Key: key, 68 | Metadata: object.metadata, 69 | }); 70 | await request.promise(); 71 | } 72 | 73 | public async deleteObject(key: string, bucket: string): Promise { 74 | await this.client.deleteObject({ Key: key, Bucket: bucket }).promise(); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/integration_tests/main.test.ts: -------------------------------------------------------------------------------- 1 | import { Storage } from '@google-cloud/storage'; 2 | import { config as loadDotEnvVars } from 'dotenv'; 3 | import { get as getEnvVar } from 'env-var'; 4 | import { Client as MinioClient } from 'minio'; 5 | 6 | import { asyncIterableToArray } from '../lib/_test_utils'; 7 | import { AdapterType } from '../lib/adapters'; 8 | import { OBJECT, OBJECT1_KEY, OBJECT_PREFIX } from '../lib/adapters/_test_utils'; 9 | import { initObjectStoreClient } from '../lib/init'; 10 | 11 | loadDotEnvVars(); 12 | 13 | const OBJECT_STORE_HOST = '127.0.0.1'; 14 | const OBJECT_STORE_BUCKET = getEnvVar('OBJECT_STORE_BUCKET').required().asString(); 15 | const OBJECT_STORE_ACCESS_KEY_ID = getEnvVar('OBJECT_STORE_ACCESS_KEY_ID').required().asString(); 16 | const OBJECT_STORE_SECRET_KEY = getEnvVar('OBJECT_STORE_SECRET_KEY').required().asString(); 17 | 18 | test('Minio', async () => { 19 | const minioPort = 9000; 20 | const minioClient = new MinioClient({ 21 | accessKey: OBJECT_STORE_ACCESS_KEY_ID, 22 | endPoint: OBJECT_STORE_HOST, 23 | port: minioPort, 24 | secretKey: OBJECT_STORE_SECRET_KEY, 25 | useSSL: false, 26 | }); 27 | await minioClient.makeBucket(OBJECT_STORE_BUCKET, ''); 28 | 29 | await testClient('minio', minioPort); 30 | }); 31 | 32 | test('GCS', async () => { 33 | const gcsPort = 8080; 34 | const gcsClient = new Storage({ 35 | apiEndpoint: `http://${OBJECT_STORE_HOST}:${gcsPort}`, 36 | projectId: 'this is not actually used, but we have to set it :shrugs:', 37 | }); 38 | await gcsClient.createBucket(OBJECT_STORE_BUCKET); 39 | 40 | await testClient('gcs', gcsPort, false); 41 | }); 42 | 43 | async function testClient( 44 | adapterType: AdapterType, 45 | port: number, 46 | passCredentials = true, 47 | ): Promise { 48 | const client = initObjectStoreClient( 49 | adapterType, 50 | `${OBJECT_STORE_HOST}:${port}`, 51 | passCredentials ? OBJECT_STORE_ACCESS_KEY_ID : undefined, 52 | passCredentials ? OBJECT_STORE_SECRET_KEY : undefined, 53 | false, 54 | ); 55 | 56 | await expect( 57 | asyncIterableToArray(client.listObjectKeys(OBJECT_PREFIX, OBJECT_STORE_BUCKET)), 58 | ).resolves.toHaveLength(0); 59 | // Retrieving an object that doesn't exist yet 60 | await expect(client.getObject(OBJECT1_KEY, OBJECT_STORE_BUCKET)).resolves.toBeNull(); 61 | 62 | await client.putObject(OBJECT, OBJECT1_KEY, OBJECT_STORE_BUCKET); 63 | 64 | await expect( 65 | asyncIterableToArray(client.listObjectKeys(OBJECT_PREFIX, OBJECT_STORE_BUCKET)), 66 | ).resolves.toEqual([OBJECT1_KEY]); 67 | 68 | await expect(client.getObject(OBJECT1_KEY, OBJECT_STORE_BUCKET)).resolves.toEqual(OBJECT); 69 | 70 | await expect(client.deleteObject(OBJECT1_KEY, OBJECT_STORE_BUCKET)).toResolve(); 71 | await expect( 72 | asyncIterableToArray(client.listObjectKeys(OBJECT_PREFIX, OBJECT_STORE_BUCKET)), 73 | ).resolves.toHaveLength(0); 74 | 75 | await expect(client.deleteObject('non-existing.txt', OBJECT_STORE_BUCKET)).toResolve(); 76 | } 77 | -------------------------------------------------------------------------------- /src/lib/init.spec.ts: -------------------------------------------------------------------------------- 1 | import { ACCESS_KEY, ENDPOINT, SECRET_ACCESS_KEY } from './_test_utils'; 2 | import { AdapterType, CLIENT_BY_ADAPTER_NAME } from './adapters'; 3 | import { S3Client } from './adapters/S3Client'; 4 | import { ObjectStorageError } from './errors'; 5 | import { initObjectStoreClient } from './init'; 6 | 7 | jest.mock('./adapters/S3Client'); 8 | jest.mock('./adapters/GCSClient'); 9 | 10 | const ADAPTERS_WITHOUT_CREDENTIALS_SUPPORT: readonly AdapterType[] = ['gcs']; 11 | 12 | describe('initObjectStoreClient', () => { 13 | test('An invalid type should be refused', () => { 14 | const invalidType = 'foo'; 15 | expect(() => 16 | initObjectStoreClient(invalidType as any, ENDPOINT, ACCESS_KEY, SECRET_ACCESS_KEY), 17 | ).toThrowError(new ObjectStorageError(`Unknown client type "${invalidType}"`)); 18 | }); 19 | 20 | test.each(Object.getOwnPropertyNames(CLIENT_BY_ADAPTER_NAME))( 21 | '%s client should be returned if requested', 22 | (adapterName) => { 23 | const passCredentials = !ADAPTERS_WITHOUT_CREDENTIALS_SUPPORT.includes( 24 | adapterName as AdapterType, 25 | ); 26 | const client = initObjectStoreClient( 27 | adapterName as any, 28 | ENDPOINT, 29 | passCredentials ? ACCESS_KEY : undefined, 30 | passCredentials ? SECRET_ACCESS_KEY : undefined, 31 | ); 32 | 33 | const expectedClientClass = CLIENT_BY_ADAPTER_NAME[adapterName as AdapterType]; 34 | expect(client).toBeInstanceOf(expectedClientClass); 35 | const credentials = { 36 | accessKeyId: ACCESS_KEY, 37 | secretAccessKey: SECRET_ACCESS_KEY, 38 | }; 39 | expect(expectedClientClass).toBeCalledWith({ 40 | endpoint: ENDPOINT, 41 | tlsEnabled: true, 42 | ...(passCredentials && { credentials }), 43 | }); 44 | }, 45 | ); 46 | 47 | test('Endpoint should be unset if absent', () => { 48 | initObjectStoreClient('s3'); 49 | 50 | expect(S3Client).toBeCalledWith( 51 | expect.not.objectContaining({ 52 | endpoint: expect.anything(), 53 | }), 54 | ); 55 | }); 56 | 57 | test('Credentials should be unset if absent', () => { 58 | initObjectStoreClient('s3'); 59 | 60 | expect(S3Client).toBeCalledWith( 61 | expect.not.objectContaining({ 62 | credentials: expect.anything(), 63 | }), 64 | ); 65 | }); 66 | 67 | test('Partially setting credentials should be refused', () => { 68 | expect(() => initObjectStoreClient('s3', ENDPOINT, ACCESS_KEY, undefined)).toThrow( 69 | new ObjectStorageError( 70 | 'Both access key id and secret access key must be set, or neither must be set', 71 | ), 72 | ); 73 | expect(() => initObjectStoreClient('s3', ENDPOINT, undefined, SECRET_ACCESS_KEY)).toThrow( 74 | new ObjectStorageError( 75 | 'Both access key id and secret access key must be set, or neither must be set', 76 | ), 77 | ); 78 | }); 79 | 80 | test('TLS should be disabled if requested', () => { 81 | initObjectStoreClient('s3', ENDPOINT, ACCESS_KEY, SECRET_ACCESS_KEY, false); 82 | 83 | expect(S3Client).toBeCalledWith( 84 | expect.objectContaining({ 85 | tlsEnabled: false, 86 | }), 87 | ); 88 | }); 89 | }); 90 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | // For a detailed explanation regarding each configuration property, visit: 2 | // https://jestjs.io/docs/en/configuration.html 3 | 4 | module.exports = { 5 | // All imported modules in your tests should be mocked automatically 6 | // automock: false, 7 | 8 | // Stop running tests after `n` failures 9 | // bail: 0, 10 | 11 | // Respect "browser" field in package.json when resolving modules 12 | // browser: false, 13 | 14 | // The directory where Jest should store its cached dependency information 15 | // cacheDirectory: "/tmp/jest_rs", 16 | 17 | // Automatically clear mock calls and instances between every test 18 | clearMocks: true, 19 | 20 | // Disable coverage here as a workaround to make debugging work in WebStorm 21 | // Coverage should still be enabled in package.json scripts. 22 | collectCoverage: false, 23 | 24 | // An array of glob patterns indicating a set of files for which coverage information should be collected 25 | collectCoverageFrom: ["src/lib/**/*.ts"], 26 | 27 | // The directory where Jest should output its coverage files 28 | coverageDirectory: "coverage", 29 | 30 | // An array of regexp pattern strings used to skip coverage collection 31 | coveragePathIgnorePatterns: [ 32 | "_test_utils\.[tj]s", 33 | ], 34 | 35 | // A list of reporter names that Jest uses when writing coverage reports 36 | coverageReporters: [ 37 | "text", 38 | "lcov", 39 | ], 40 | 41 | // An object that configures minimum threshold enforcement for coverage results 42 | coverageThreshold: { 43 | "global": { 44 | "branches": 100, 45 | "functions": 100, 46 | "lines": 100, 47 | "statements": 100 48 | } 49 | }, 50 | 51 | // A path to a custom dependency extractor 52 | // dependencyExtractor: null, 53 | 54 | // Make calling deprecated APIs throw helpful error messages 55 | // errorOnDeprecated: false, 56 | 57 | // Force coverage collection from ignored files using an array of glob patterns 58 | // forceCoverageMatch: [], 59 | 60 | // A path to a module which exports an async function that is triggered once before all test suites 61 | // globalSetup: null, 62 | 63 | // A path to a module which exports an async function that is triggered once after all test suites 64 | // globalTeardown: null, 65 | 66 | // A set of global variables that need to be available in all test environments 67 | // globals: {}, 68 | 69 | // An array of directory names to be searched recursively up from the requiring module's location 70 | // moduleDirectories: [ 71 | // "node_modules" 72 | // ], 73 | 74 | // An array of file extensions your modules use 75 | moduleFileExtensions: ["js", "ts"], 76 | 77 | // A map from regular expressions to module names that allow to stub out resources with a single module 78 | // moduleNameMapper: {}, 79 | 80 | // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader 81 | // modulePathIgnorePatterns: [], 82 | 83 | // Activates notifications for test results 84 | // notify: false, 85 | 86 | // An enum that specifies notification mode. Requires { notify: true } 87 | // notifyMode: "failure-change", 88 | 89 | // A preset that is used as a base for Jest's configuration 90 | preset: "ts-jest", 91 | 92 | // Run tests from one or more projects 93 | // projects: null, 94 | 95 | // Use this configuration option to add custom reporters to Jest 96 | // reporters: undefined, 97 | 98 | // Automatically reset mock state between every test 99 | // resetMocks: false, 100 | 101 | // Reset the module registry before running each individual test 102 | // resetModules: false, 103 | 104 | // A path to a custom resolver 105 | // resolver: null, 106 | 107 | // Automatically restore mock state between every test 108 | // restoreMocks: false, 109 | 110 | // The root directory that Jest should scan for tests and modules within 111 | // rootDir: null, 112 | 113 | // A list of paths to directories that Jest should use to search for files in 114 | roots: ['src/lib'], 115 | 116 | // Allows you to use a custom runner instead of Jest's default test runner 117 | // runner: "jest-runner", 118 | 119 | // The paths to modules that run some code to configure or set up the testing environment before each test 120 | setupFiles: ["jest-date-mock"], 121 | 122 | // A list of paths to modules that run some code to configure or set up the testing framework before each test 123 | setupFilesAfterEnv: ["jest-extended/all"], 124 | 125 | // A list of paths to snapshot serializer modules Jest should use for snapshot testing 126 | // snapshotSerializers: [], 127 | 128 | // The test environment that will be used for testing 129 | testEnvironment: "node", 130 | 131 | // Options that will be passed to the testEnvironment 132 | // testEnvironmentOptions: {}, 133 | 134 | // Adds a location field to test results 135 | // testLocationInResults: false, 136 | 137 | // The glob patterns Jest uses to detect test files 138 | // testMatch: [ 139 | // "**/__tests__/**/*.[jt]s?(x)", 140 | // "**/?(*.)+(spec|test).[tj]s?(x)" 141 | // ], 142 | 143 | // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped 144 | testPathIgnorePatterns: [ 145 | "/node_modules/", 146 | ], 147 | 148 | // The regexp pattern or array of patterns that Jest uses to detect test files 149 | // testRegex: [], 150 | 151 | // This option allows the use of a custom results processor 152 | // testResultsProcessor: null, 153 | 154 | // This option allows use of a custom test runner 155 | // testRunner: "jasmine2", 156 | 157 | // This option sets the URL for the jsdom environment. It is reflected in properties such as location.href 158 | // testURL: "http://localhost", 159 | 160 | // Setting this value to "fake" allows the use of fake timers for functions such as "setTimeout" 161 | // timers: "real", 162 | 163 | // A map from regular expressions to paths to transformers 164 | // transform: null, 165 | 166 | // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation 167 | // transformIgnorePatterns: [ 168 | // "/node_modules/" 169 | // ], 170 | 171 | // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them 172 | // unmockedModulePathPatterns: undefined, 173 | 174 | // Indicates whether each individual test should be reported during the run 175 | // verbose: null, 176 | 177 | // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode 178 | // watchPathIgnorePatterns: [], 179 | 180 | // Whether to use watchman for file crawling 181 | // watchman: true, 182 | }; 183 | -------------------------------------------------------------------------------- /src/lib/adapters/GCSClient.spec.ts: -------------------------------------------------------------------------------- 1 | import { Storage } from '@google-cloud/storage'; 2 | 3 | import { 4 | asyncIterableToArray, 5 | CLIENT_CREDENTIALS, 6 | ENDPOINT, 7 | getPromiseRejection, 8 | mockSpy, 9 | } from '../_test_utils'; 10 | import { ObjectStorageError } from '../errors'; 11 | import { 12 | BUCKET, 13 | CLIENT_CONFIG, 14 | OBJECT, 15 | OBJECT1_KEY, 16 | OBJECT2_KEY, 17 | OBJECT_PREFIX, 18 | } from './_test_utils'; 19 | import { GCSClient } from './GCSClient'; 20 | 21 | const mockFile: any = { 22 | delete: mockSpy(jest.fn()), 23 | download: mockSpy(jest.fn(), () => Promise.resolve([OBJECT.body])), 24 | get: mockSpy(jest.fn(), () => Promise.resolve([mockFile, {}])), 25 | metadata: { metadata: OBJECT.metadata }, 26 | save: mockSpy(jest.fn()), 27 | setMetadata: mockSpy(jest.fn()), 28 | }; 29 | const mockBucket = { 30 | file: mockSpy(jest.fn(), () => mockFile), 31 | getFiles: mockSpy(jest.fn(), () => Promise.resolve([[], null])), 32 | }; 33 | const mockStorage = { 34 | bucket: mockSpy(jest.fn(), (bucketName) => { 35 | expect(bucketName).toEqual(BUCKET); 36 | return mockBucket; 37 | }), 38 | }; 39 | jest.mock('@google-cloud/storage', () => ({ 40 | Storage: jest.fn().mockImplementation(() => mockStorage), 41 | })); 42 | 43 | const CLIENT = new GCSClient(CLIENT_CONFIG); 44 | 45 | describe('Constructor', () => { 46 | test('Specified endpoint should be specified', () => { 47 | // tslint:disable-next-line:no-unused-expression 48 | new GCSClient(CLIENT_CONFIG); 49 | 50 | expect(Storage).toBeCalledWith(expect.objectContaining({ apiEndpoint: `https://${ENDPOINT}` })); 51 | }); 52 | 53 | test('No endpoint should be specified if none was passed', () => { 54 | // tslint:disable-next-line:no-unused-expression 55 | new GCSClient({ tlsEnabled: true }); 56 | 57 | expect(Storage).toBeCalledWith(expect.not.objectContaining({ apiEndpoint: expect.anything() })); 58 | }); 59 | 60 | test('HMAC key credentials should be refused', () => { 61 | expect(() => new GCSClient({ ...CLIENT_CONFIG, credentials: CLIENT_CREDENTIALS })).toThrow( 62 | new ObjectStorageError('GCS adapter does not accept HMAC key credentials'), 63 | ); 64 | }); 65 | 66 | test('TSL should be enabled if requested', () => { 67 | // tslint:disable-next-line:no-unused-expression 68 | new GCSClient({ endpoint: ENDPOINT, tlsEnabled: true }); 69 | 70 | expect(Storage).toBeCalledWith( 71 | expect.objectContaining({ apiEndpoint: expect.stringMatching(/^https:\/\//) }), 72 | ); 73 | }); 74 | 75 | test('TSL may be disabled if requested and custom endpoint is set', () => { 76 | // tslint:disable-next-line:no-unused-expression 77 | new GCSClient({ endpoint: ENDPOINT, tlsEnabled: false }); 78 | 79 | expect(Storage).toBeCalledWith( 80 | expect.objectContaining({ apiEndpoint: expect.stringMatching(/^http:\/\//) }), 81 | ); 82 | }); 83 | 84 | test('TSL may not be disabled if default endpoint is used', () => { 85 | // tslint:disable-next-line:no-unused-expression 86 | expect(() => new GCSClient({ tlsEnabled: false })).toThrow( 87 | new ObjectStorageError('GCS adapter requires custom endpoint if TLS is to be disabled'), 88 | ); 89 | }); 90 | }); 91 | 92 | describe('listObjectKeys', () => { 93 | test('Prefix should be honored', async () => { 94 | await asyncIterableToArray(CLIENT.listObjectKeys(OBJECT_PREFIX, BUCKET)); 95 | 96 | expect(mockBucket.getFiles).toBeCalledWith(expect.objectContaining({ prefix: OBJECT_PREFIX })); 97 | }); 98 | 99 | test('Keys for objects matching criteria should be yielded', async () => { 100 | const objectKeys: readonly string[] = [`${OBJECT_PREFIX}logo.png`, `${OBJECT_PREFIX}logo.gif`]; 101 | mockBucket.getFiles.mockResolvedValue([objectKeys.map((k) => ({ name: k })), null]); 102 | 103 | const retrievedKeys = await asyncIterableToArray(CLIENT.listObjectKeys(OBJECT_PREFIX, BUCKET)); 104 | 105 | expect(retrievedKeys).toEqual(objectKeys); 106 | }); 107 | 108 | test('Pagination should be handled seamlessly', async () => { 109 | const objectKeys1: readonly string[] = [OBJECT1_KEY, OBJECT2_KEY]; 110 | const page2Query = { foo: 'bar' }; 111 | mockBucket.getFiles.mockResolvedValueOnce([objectKeys1.map((k) => ({ name: k })), page2Query]); 112 | const objectKeys2: readonly string[] = [ 113 | `${OBJECT_PREFIX}style.css`, 114 | `${OBJECT_PREFIX}mobile.css`, 115 | ]; 116 | mockBucket.getFiles.mockResolvedValueOnce([objectKeys2.map((k) => ({ name: k })), null]); 117 | 118 | const retrievedKeys = await asyncIterableToArray(CLIENT.listObjectKeys(OBJECT_PREFIX, BUCKET)); 119 | 120 | expect(retrievedKeys).toEqual([...objectKeys1, ...objectKeys2]); 121 | 122 | expect(mockBucket.getFiles).nthCalledWith(1, expect.objectContaining({ autoPaginate: false })); 123 | expect(mockBucket.getFiles).nthCalledWith(2, page2Query); 124 | }); 125 | }); 126 | 127 | describe('getObject', () => { 128 | test('Object should be retrieved with the specified key', async () => { 129 | await CLIENT.getObject(OBJECT1_KEY, BUCKET); 130 | 131 | expect(mockBucket.file).toBeCalledWith(OBJECT1_KEY); 132 | }); 133 | 134 | test('Body and metadata should be output', async () => { 135 | const object = await CLIENT.getObject(OBJECT1_KEY, BUCKET); 136 | 137 | expect(object).toHaveProperty('body', OBJECT.body); 138 | expect(object).toHaveProperty('metadata', OBJECT.metadata); 139 | }); 140 | 141 | test('Metadata should fall back to empty object when undefined', async () => { 142 | mockFile.get.mockResolvedValue([{ download: mockFile.download, metadata: {} }]); 143 | 144 | const object = await CLIENT.getObject(OBJECT1_KEY, BUCKET); 145 | 146 | expect(object).toHaveProperty('body', OBJECT.body); 147 | expect(object).toHaveProperty('metadata', {}); 148 | }); 149 | 150 | test('Nothing should be returned if the key does not exist', async () => { 151 | mockFile.get.mockRejectedValue(new ApiError('Whoops', 404)); 152 | 153 | await expect(CLIENT.getObject(OBJECT1_KEY, BUCKET)).resolves.toBeNull(); 154 | }); 155 | 156 | test('Errors other than a missing key should be wrapped', async () => { 157 | const apiError = new Error('Whoops'); 158 | mockFile.get.mockRejectedValue(apiError); 159 | 160 | const error = await getPromiseRejection( 161 | CLIENT.getObject(OBJECT1_KEY, BUCKET), 162 | ObjectStorageError, 163 | ); 164 | 165 | expect(error.message).toStartWith('Failed to retrieve object'); 166 | expect(error.cause()).toEqual(apiError); 167 | }); 168 | }); 169 | 170 | describe('putObject', () => { 171 | test('Object should be created with specified parameters', async () => { 172 | await CLIENT.putObject(OBJECT, OBJECT1_KEY, BUCKET); 173 | 174 | expect(mockBucket.file).toBeCalledWith(OBJECT1_KEY); 175 | expect(mockFile.save).toBeCalledWith(OBJECT.body, { resumable: false }); 176 | expect(mockFile.setMetadata).toBeCalledWith({ metadata: OBJECT.metadata }); 177 | }); 178 | }); 179 | 180 | describe('deleteObject', () => { 181 | test('Specified object should be deleted', async () => { 182 | await CLIENT.deleteObject(OBJECT1_KEY, BUCKET); 183 | 184 | expect(mockBucket.file).toBeCalledWith(OBJECT1_KEY); 185 | expect(mockFile.delete).toBeCalledWith(); 186 | }); 187 | 188 | test('Failure to delete non-existing object should be suppressed', async () => { 189 | mockFile.delete.mockRejectedValue(new ApiError('Whoops', 404)); 190 | 191 | await expect(CLIENT.deleteObject(OBJECT1_KEY, BUCKET)).toResolve(); 192 | }); 193 | 194 | test('Other errors should be wrapped', async () => { 195 | const apiError = new Error('Whoops'); 196 | mockFile.delete.mockRejectedValue(apiError); 197 | 198 | const error = await getPromiseRejection( 199 | CLIENT.deleteObject(OBJECT1_KEY, BUCKET), 200 | ObjectStorageError, 201 | ); 202 | 203 | expect(error.message).toStartWith('Failed to delete object'); 204 | expect(error.cause()).toEqual(apiError); 205 | }); 206 | }); 207 | 208 | /** 209 | * Mock GCS' own `ApiError`. 210 | */ 211 | class ApiError extends Error { 212 | constructor(message: string, public readonly code: number) { 213 | super(message); 214 | } 215 | } 216 | -------------------------------------------------------------------------------- /src/lib/adapters/S3Client.spec.ts: -------------------------------------------------------------------------------- 1 | import * as http from 'http'; 2 | import * as https from 'https'; 3 | 4 | import { 5 | ACCESS_KEY, 6 | asyncIterableToArray, 7 | CLIENT_CREDENTIALS, 8 | ENDPOINT, 9 | getMockContext, 10 | mockSpy, 11 | SECRET_ACCESS_KEY, 12 | } from '../_test_utils'; 13 | import { 14 | BUCKET, 15 | CLIENT_CONFIG, 16 | OBJECT, 17 | OBJECT1_KEY, 18 | OBJECT2_KEY, 19 | OBJECT_PREFIX, 20 | } from './_test_utils'; 21 | 22 | const mockS3Client = { 23 | deleteObject: mockSpy(jest.fn(), () => ({ promise: () => Promise.resolve() })), 24 | getObject: mockSpy(jest.fn(), () => ({ promise: () => Promise.resolve({}) })), 25 | listObjectsV2: mockSpy(jest.fn(), () => ({ promise: () => Promise.resolve({ Contents: [] }) })), 26 | putObject: mockSpy(jest.fn(), () => ({ promise: () => Promise.resolve() })), 27 | }; 28 | jest.mock('aws-sdk', () => ({ 29 | S3: jest.fn().mockImplementation(() => mockS3Client), 30 | })); 31 | import * as AWS from 'aws-sdk'; 32 | import { S3Client } from './S3Client'; 33 | 34 | const CLIENT = new S3Client(CLIENT_CONFIG); 35 | 36 | describe('Constructor', () => { 37 | test('Specified endpoint should be used', () => { 38 | // tslint:disable-next-line:no-unused-expression 39 | new S3Client(CLIENT_CONFIG); 40 | 41 | expect(AWS.S3).toBeCalledTimes(1); 42 | 43 | const s3CallArgs = getMockContext(AWS.S3).calls[0][0]; 44 | expect(s3CallArgs).toHaveProperty('endpoint', ENDPOINT); 45 | }); 46 | 47 | test('Endpoint should be skipped if unspecified', () => { 48 | // tslint:disable-next-line:no-unused-expression 49 | new S3Client({ tlsEnabled: true }); 50 | 51 | expect(AWS.S3).toBeCalledTimes(1); 52 | 53 | const s3CallArgs = getMockContext(AWS.S3).calls[0][0]; 54 | expect(s3CallArgs).not.toHaveProperty('endpoint'); 55 | }); 56 | 57 | test('Specified credentials should be used', () => { 58 | // tslint:disable-next-line:no-unused-expression 59 | new S3Client({ ...CLIENT_CONFIG, credentials: CLIENT_CREDENTIALS }); 60 | 61 | expect(AWS.S3).toBeCalledTimes(1); 62 | 63 | const s3CallArgs = getMockContext(AWS.S3).calls[0][0]; 64 | expect(s3CallArgs).toHaveProperty('accessKeyId', ACCESS_KEY); 65 | expect(s3CallArgs).toHaveProperty('secretAccessKey', SECRET_ACCESS_KEY); 66 | }); 67 | 68 | test('Credentials should be skipped if unspecified', () => { 69 | // tslint:disable-next-line:no-unused-expression 70 | new S3Client(CLIENT_CONFIG); 71 | 72 | expect(AWS.S3).toBeCalledTimes(1); 73 | 74 | const s3CallArgs = getMockContext(AWS.S3).calls[0][0]; 75 | expect(s3CallArgs).not.toHaveProperty('accessKeyId'); 76 | expect(s3CallArgs).not.toHaveProperty('secretAccessKey'); 77 | }); 78 | 79 | test('Signature should use version 4', () => { 80 | // tslint:disable-next-line:no-unused-expression 81 | new S3Client(CLIENT_CONFIG); 82 | 83 | expect(AWS.S3).toBeCalledTimes(1); 84 | 85 | const s3CallArgs = getMockContext(AWS.S3).calls[0][0]; 86 | expect(s3CallArgs).toHaveProperty('signatureVersion', 'v4'); 87 | }); 88 | 89 | test('s3ForcePathStyle should be enabled', () => { 90 | // tslint:disable-next-line:no-unused-expression 91 | new S3Client(CLIENT_CONFIG); 92 | 93 | expect(AWS.S3).toBeCalledTimes(1); 94 | 95 | const s3CallArgs = getMockContext(AWS.S3).calls[0][0]; 96 | expect(s3CallArgs).toHaveProperty('s3ForcePathStyle', true); 97 | }); 98 | 99 | test('TSL should be enabled if requested', () => { 100 | // tslint:disable-next-line:no-unused-expression 101 | new S3Client({ ...CLIENT_CONFIG, tlsEnabled: true }); 102 | 103 | expect(AWS.S3).toBeCalledTimes(1); 104 | 105 | const s3CallArgs = getMockContext(AWS.S3).calls[0][0]; 106 | expect(s3CallArgs).toHaveProperty('sslEnabled', true); 107 | }); 108 | 109 | test('TSL may be disabled if requested', () => { 110 | // tslint:disable-next-line:no-unused-expression 111 | new S3Client({ ...CLIENT_CONFIG, tlsEnabled: false }); 112 | 113 | expect(AWS.S3).toBeCalledTimes(1); 114 | 115 | const s3CallArgs = getMockContext(AWS.S3).calls[0][0]; 116 | expect(s3CallArgs).toHaveProperty('sslEnabled', false); 117 | }); 118 | 119 | describe('HTTP(S) agent', () => { 120 | test('HTTP agent with Keep-Alive should be used when TSL is disabled', () => { 121 | // tslint:disable-next-line:no-unused-expression 122 | new S3Client({ ...CLIENT_CONFIG, tlsEnabled: false }); 123 | 124 | expect(AWS.S3).toBeCalledTimes(1); 125 | 126 | const s3CallArgs = getMockContext(AWS.S3).calls[0][0]; 127 | const agent = s3CallArgs.httpOptions.agent; 128 | expect(agent).toBeInstanceOf(http.Agent); 129 | expect(agent).toHaveProperty('keepAlive', true); 130 | }); 131 | 132 | test('HTTPS agent with Keep-Alive should be used when TSL is enabled', () => { 133 | // tslint:disable-next-line:no-unused-expression 134 | new S3Client({ ...CLIENT_CONFIG, tlsEnabled: true }); 135 | 136 | expect(AWS.S3).toBeCalledTimes(1); 137 | 138 | const s3CallArgs = getMockContext(AWS.S3).calls[0][0]; 139 | const agent = s3CallArgs.httpOptions.agent; 140 | expect(agent).toBeInstanceOf(https.Agent); 141 | expect(agent).toHaveProperty('keepAlive', true); 142 | }); 143 | }); 144 | }); 145 | 146 | describe('listObjectKeys', () => { 147 | test('Prefix and bucket should be honored', async () => { 148 | await asyncIterableToArray(CLIENT.listObjectKeys(OBJECT_PREFIX, BUCKET)); 149 | 150 | expect(mockS3Client.listObjectsV2).toBeCalledTimes(1); 151 | expect(mockS3Client.listObjectsV2).toBeCalledWith({ 152 | Bucket: BUCKET, 153 | Prefix: OBJECT_PREFIX, 154 | }); 155 | }); 156 | 157 | test('Keys for objects matching criteria should be yielded', async () => { 158 | const objectKeys: readonly string[] = [OBJECT1_KEY, OBJECT2_KEY]; 159 | mockS3Client.listObjectsV2.mockReturnValue({ 160 | promise: () => 161 | Promise.resolve({ 162 | Contents: objectKeys.map((k) => ({ Key: k })), 163 | IsTruncated: false, 164 | }), 165 | }); 166 | 167 | const retrievedKeys = await asyncIterableToArray(CLIENT.listObjectKeys(OBJECT_PREFIX, BUCKET)); 168 | 169 | expect(retrievedKeys).toEqual(objectKeys); 170 | }); 171 | 172 | test('Pagination should be handled seamlessly', async () => { 173 | const objectKeys1: readonly string[] = [OBJECT1_KEY, OBJECT2_KEY]; 174 | const continuationToken = 'continue=='; 175 | mockS3Client.listObjectsV2.mockReturnValueOnce({ 176 | promise: () => 177 | Promise.resolve({ 178 | Contents: objectKeys1.map((k) => ({ Key: k })), 179 | IsTruncated: true, 180 | NextContinuationToken: continuationToken, 181 | }), 182 | }); 183 | const objectKeys2: readonly string[] = [ 184 | `${OBJECT_PREFIX}style.css`, 185 | `${OBJECT_PREFIX}mobile.css`, 186 | ]; 187 | mockS3Client.listObjectsV2.mockReturnValueOnce({ 188 | promise: () => 189 | Promise.resolve({ 190 | Contents: objectKeys2.map((k) => ({ Key: k })), 191 | IsTruncated: false, 192 | }), 193 | }); 194 | 195 | const retrievedKeys = await asyncIterableToArray(CLIENT.listObjectKeys(OBJECT_PREFIX, BUCKET)); 196 | 197 | expect(retrievedKeys).toEqual([...objectKeys1, ...objectKeys2]); 198 | 199 | expect(mockS3Client.listObjectsV2).toBeCalledTimes(2); 200 | expect(mockS3Client.listObjectsV2).toBeCalledWith({ 201 | Bucket: BUCKET, 202 | ContinuationToken: continuationToken, 203 | Prefix: OBJECT_PREFIX, 204 | }); 205 | }); 206 | }); 207 | 208 | describe('getObject', () => { 209 | test('Object should be retrieved with the specified parameters', async () => { 210 | await CLIENT.getObject(OBJECT1_KEY, BUCKET); 211 | 212 | expect(mockS3Client.getObject).toBeCalledTimes(1); 213 | expect(mockS3Client.getObject).toBeCalledWith({ 214 | Bucket: BUCKET, 215 | Key: OBJECT1_KEY, 216 | }); 217 | }); 218 | 219 | test('Body and metadata should be output', async () => { 220 | mockS3Client.getObject.mockReturnValue({ 221 | promise: () => 222 | Promise.resolve({ 223 | Body: OBJECT.body, 224 | Metadata: OBJECT.metadata, 225 | }), 226 | }); 227 | 228 | const object = await CLIENT.getObject(OBJECT1_KEY, BUCKET); 229 | 230 | expect(object).toHaveProperty('body', OBJECT.body); 231 | expect(object).toHaveProperty('metadata', OBJECT.metadata); 232 | }); 233 | 234 | test('Metadata should fall back to empty object when undefined', async () => { 235 | mockS3Client.getObject.mockReturnValue({ 236 | promise: () => 237 | Promise.resolve({ 238 | Body: OBJECT.body, 239 | }), 240 | }); 241 | 242 | const object = await CLIENT.getObject(OBJECT1_KEY, BUCKET); 243 | 244 | expect(object).toHaveProperty('body', OBJECT.body); 245 | expect(object).toHaveProperty('metadata', {}); 246 | }); 247 | 248 | test('Nothing should be returned if the key does not exist', async () => { 249 | mockS3Client.getObject.mockReturnValue({ 250 | promise: () => Promise.reject({ code: 'NoSuchKey' }), 251 | }); 252 | 253 | const object = await CLIENT.getObject(OBJECT1_KEY, BUCKET); 254 | 255 | expect(object).toBeNull(); 256 | }); 257 | 258 | test('Errors other than NoSuchKey should be propagated', async () => { 259 | const error = new Error('Oh noes'); 260 | mockS3Client.getObject.mockReturnValue({ 261 | promise: () => Promise.reject(error), 262 | }); 263 | 264 | await expect(CLIENT.getObject(OBJECT1_KEY, BUCKET)).rejects.toEqual(error); 265 | }); 266 | }); 267 | 268 | describe('putObject', () => { 269 | test('Object should be created with specified parameters', async () => { 270 | await CLIENT.putObject(OBJECT, OBJECT1_KEY, BUCKET); 271 | 272 | expect(mockS3Client.putObject).toBeCalledTimes(1); 273 | expect(mockS3Client.putObject).toBeCalledWith({ 274 | Body: OBJECT.body, 275 | Bucket: BUCKET, 276 | Key: OBJECT1_KEY, 277 | Metadata: OBJECT.metadata, 278 | }); 279 | }); 280 | }); 281 | 282 | describe('deleteObject', () => { 283 | test('Specified object should be deleted', async () => { 284 | await CLIENT.deleteObject(OBJECT1_KEY, BUCKET); 285 | 286 | expect(mockS3Client.deleteObject).toBeCalledTimes(1); 287 | expect(mockS3Client.deleteObject).toBeCalledWith({ 288 | Bucket: BUCKET, 289 | Key: OBJECT1_KEY, 290 | }); 291 | }); 292 | }); 293 | --------------------------------------------------------------------------------