├── .github ├── dependabot.yml └── workflows │ ├── codeql.yaml │ ├── dependency-review.yaml │ └── tests.yaml ├── .gitignore ├── .npmignore ├── .yamllint ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── babel.config.js ├── circle.yml ├── documentation ├── BucketInfoModelVersion.md └── listingAlgos │ ├── delimiter.md │ ├── delimiterMaster.md │ ├── delimiterVersions.md │ └── pics │ ├── delimiterMasterV0StateChart.dot │ ├── delimiterMasterV0StateChart.svg │ ├── delimiterStateChart.dot │ ├── delimiterStateChart.svg │ ├── delimiterVersionsStateChart.dot │ └── delimiterVersionsStateChart.svg ├── eslint.config.mjs ├── greenkeeper.json ├── index.ts ├── lib ├── Clustering.ts ├── algos │ ├── cache │ │ ├── GapCache.ts │ │ ├── GapSet.ts │ │ └── LRUCache.js │ ├── heap │ │ └── Heap.ts │ ├── list │ │ ├── Extension.js │ │ ├── MPU.js │ │ ├── basic.js │ │ ├── delimiter.ts │ │ ├── delimiterCurrent.ts │ │ ├── delimiterMaster.ts │ │ ├── delimiterNonCurrent.js │ │ ├── delimiterOrphanDeleteMarker.js │ │ ├── delimiterVersions.ts │ │ ├── exportAlgos.js │ │ ├── skip.js │ │ └── tools.js │ ├── set │ │ ├── ArrayUtils.js │ │ └── SortedSet.js │ └── stream │ │ └── MergeStream.js ├── auth │ ├── AuthInfo.ts │ ├── Vault.ts │ ├── auth.ts │ ├── backends │ │ ├── ChainBackend.ts │ │ ├── base.ts │ │ └── in_memory │ │ │ ├── AuthLoader.ts │ │ │ ├── Backend.ts │ │ │ ├── Indexer.ts │ │ │ ├── types.ts │ │ │ ├── validateAuthConfig.ts │ │ │ └── vaultUtilities.ts │ ├── v2 │ │ ├── algoCheck.ts │ │ ├── authV2.ts │ │ ├── checkRequestExpiry.ts │ │ ├── constructStringToSign.ts │ │ ├── getCanonicalizedAmzHeaders.ts │ │ ├── getCanonicalizedResource.ts │ │ ├── headerAuthCheck.ts │ │ └── queryAuthCheck.ts │ └── v4 │ │ ├── authV4.ts │ │ ├── awsURIencode.ts │ │ ├── constructStringToSign.ts │ │ ├── createCanonicalRequest.ts │ │ ├── headerAuthCheck.ts │ │ ├── queryAuthCheck.ts │ │ ├── streamingV4 │ │ ├── V4Transform.ts │ │ └── constructChunkStringToSign.ts │ │ ├── timeUtils.ts │ │ └── validateInputs.ts ├── clustering │ └── ClusterRPC.ts ├── constants.ts ├── db.ts ├── errorUtils.ts ├── errors │ ├── arsenalErrors.ts │ └── index.ts ├── executables │ └── pensieveCreds │ │ ├── README.md │ │ ├── getPensieveCreds.js │ │ ├── package.json │ │ ├── tests │ │ ├── resources.json │ │ └── unit │ │ │ └── utilsSpec.js │ │ └── utils.js ├── https │ ├── ciphers.ts │ ├── dh2048.ts │ └── index.ts ├── ipCheck.ts ├── jsutil.ts ├── metrics │ ├── RedisClient.ts │ ├── StatsClient.ts │ ├── StatsModel.ts │ ├── ZenkoMetrics.ts │ └── index.ts ├── models │ ├── ARN.ts │ ├── BackendInfo.ts │ ├── BucketAzureInfo.ts │ ├── BucketInfo.ts │ ├── BucketPolicy.ts │ ├── LifecycleConfiguration.ts │ ├── LifecycleRule.ts │ ├── NotificationConfiguration.ts │ ├── ObjectLockConfiguration.ts │ ├── ObjectMD.ts │ ├── ObjectMDAmzRestore.ts │ ├── ObjectMDArchive.ts │ ├── ObjectMDAzureInfo.ts │ ├── ObjectMDLocation.ts │ ├── ReplicationConfiguration.ts │ ├── Veeam.ts │ ├── WebsiteConfiguration.ts │ └── index.ts ├── network │ ├── KMSInterface.ts │ ├── RoundRobin.ts │ ├── http │ │ ├── server.ts │ │ └── utils.ts │ ├── index.ts │ ├── kmip │ │ ├── Client.ts │ │ ├── ClusterClient.ts │ │ ├── Message.ts │ │ ├── README.md │ │ ├── codec │ │ │ ├── README.md │ │ │ └── ttlv.ts │ │ ├── error-comparison.jsonc │ │ ├── errorMapping.ts │ │ ├── index.ts │ │ ├── tags.json │ │ └── transport │ │ │ ├── TransportTemplate.ts │ │ │ └── tls.ts │ ├── kmsAWS │ │ ├── Client.ts │ │ └── README.md │ ├── probe │ │ ├── HealthProbeServer.ts │ │ ├── ProbeServer.ts │ │ └── Utils.ts │ ├── rest │ │ ├── RESTClient.ts │ │ ├── RESTServer.ts │ │ └── utils.ts │ ├── rpc │ │ ├── level-net.ts │ │ ├── rpc.ts │ │ ├── sio-stream.ts │ │ └── utils.ts │ └── utils.ts ├── patches │ └── locationConstraints.ts ├── policy │ ├── policyValidator.ts │ ├── resourcePolicySchema.json │ └── userPolicySchema.json ├── policyEvaluator │ ├── RequestContext.ts │ ├── evaluator.ts │ ├── principal.ts │ ├── requestUtils.ts │ └── utils │ │ ├── actionMaps.ts │ │ ├── checkArnMatch.ts │ │ ├── conditions.ts │ │ ├── objectTags.ts │ │ ├── variables.ts │ │ └── wildcards.ts ├── s3middleware │ ├── MD5Sum.ts │ ├── azureHelpers │ │ ├── ResultsCollector.ts │ │ ├── SubStreamInterface.ts │ │ └── mpuUtils.ts │ ├── convertToXml.ts │ ├── escapeForXml.ts │ ├── lifecycleHelpers │ │ ├── LifecycleDateTime.ts │ │ ├── LifecycleUtils.ts │ │ └── index.ts │ ├── nullStream.ts │ ├── objectLegalHold.ts │ ├── objectRestore.ts │ ├── objectRetention.ts │ ├── objectUtils.ts │ ├── prepareStream.ts │ ├── processMpuParts.ts │ ├── tagging.ts │ ├── userMetadata.ts │ └── validateConditionalHeaders.ts ├── s3routes │ ├── index.ts │ ├── routes.ts │ ├── routes │ │ ├── routeDELETE.ts │ │ ├── routeGET.ts │ │ ├── routeHEAD.ts │ │ ├── routeOPTIONS.ts │ │ ├── routePOST.ts │ │ ├── routePUT.ts │ │ └── routeWebsite.ts │ └── routesUtils.ts ├── shuffle.ts ├── simple-glob.d.ts ├── storage │ ├── data │ │ ├── DataWrapper.js │ │ ├── LocationConstraintParser.js │ │ ├── MultipleBackendGateway.js │ │ ├── external │ │ │ ├── AwsClient.js │ │ │ ├── AzureClient.js │ │ │ ├── GCP │ │ │ │ ├── GcpApis │ │ │ │ │ ├── abortMPU.js │ │ │ │ │ ├── completeMPU.js │ │ │ │ │ ├── createMPU.js │ │ │ │ │ ├── deleteTagging.js │ │ │ │ │ ├── getTagging.js │ │ │ │ │ ├── index.js │ │ │ │ │ ├── listParts.js │ │ │ │ │ ├── mpuHelper.js │ │ │ │ │ ├── putObject.js │ │ │ │ │ ├── putTagging.js │ │ │ │ │ ├── uploadPart.js │ │ │ │ │ └── uploadPartCopy.js │ │ │ │ ├── GcpManagedUpload.js │ │ │ │ ├── GcpService.js │ │ │ │ ├── GcpServiceSetup.js │ │ │ │ ├── GcpSigner.js │ │ │ │ ├── GcpUtils.js │ │ │ │ ├── gcp-2017-11-01.api.json │ │ │ │ └── index.js │ │ │ ├── GcpClient.js │ │ │ ├── PfsClient.js │ │ │ └── utils.js │ │ ├── file │ │ │ ├── DataFileInterface.js │ │ │ ├── DataFileStore.js │ │ │ └── utils.js │ │ ├── in_memory │ │ │ └── datastore.js │ │ └── utils │ │ │ └── RelayMD5Sum.js │ ├── metadata │ │ ├── MetadataWrapper.js │ │ ├── bucketclient │ │ │ ├── BucketClientInterface.js │ │ │ └── LogConsumer.js │ │ ├── conditions.js │ │ ├── file │ │ │ ├── BucketFileInterface.js │ │ │ ├── MetadataFileClient.js │ │ │ ├── MetadataFileServer.js │ │ │ └── RecordLog.js │ │ ├── in_memory │ │ │ ├── ListMultipartUploadsResult.js │ │ │ ├── ListResult.js │ │ │ ├── bucket_mem.js.Design.md │ │ │ ├── bucket_utilities.js │ │ │ ├── getMultipartUploadListing.js │ │ │ ├── metadata.js │ │ │ └── metastore.js │ │ ├── mongoclient │ │ │ ├── DataCounter.js │ │ │ ├── ListRecordStream.js │ │ │ ├── LogConsumer.js │ │ │ ├── MongoClientInterface.ts │ │ │ ├── Mongoclient.md │ │ │ ├── readStream.js │ │ │ └── utils.ts │ │ └── proxy │ │ │ ├── BucketdRoutes.js │ │ │ ├── README.md │ │ │ ├── Server.js │ │ │ └── utils.js │ └── utils.js ├── stream │ ├── index.ts │ └── readJSONStreamObject.ts ├── stringHash.ts ├── testing │ └── matrix.js ├── types.ts └── versioning │ ├── Version.ts │ ├── VersionID.ts │ ├── VersioningRequestProcessor.ts │ ├── WriteCache.ts │ ├── WriteGatheringManager.ts │ ├── constants.ts │ └── index.ts ├── package.json ├── tests ├── .eslintrc ├── functional │ ├── clustering │ │ ├── ClusterRPC-test-server.js │ │ ├── ClusterRPC.spec.js │ │ ├── clustering.spec.js │ │ └── utils │ │ │ ├── killed.js │ │ │ ├── shutdownTimeout.js │ │ │ ├── simple.js │ │ │ └── watchdog.js │ ├── kmip │ │ ├── highlevel.spec.js │ │ ├── lowlevel.spec.js │ │ ├── tls.spec.js │ │ └── transport.spec.js │ ├── kmsAWS │ │ └── highlevel.spec.js │ ├── metadata │ │ └── mongodb │ │ │ ├── delObject.spec.js │ │ │ ├── getBucketInfos.spec.js │ │ │ ├── getObject.spec.js │ │ │ ├── getObjects.spec.js │ │ │ ├── listLifecycleObject │ │ │ ├── current.spec.js │ │ │ ├── noncurrent.spec.js │ │ │ ├── nullVersion.spec.js │ │ │ ├── orphanDeleteMarker.spec.js │ │ │ └── utils.js │ │ │ ├── listObject.spec.js │ │ │ ├── putObject.spec.js │ │ │ └── withCond.spec.js │ ├── metadataProxy │ │ └── routesToMem.spec.js │ └── metrics │ │ ├── StatsClient.spec.js │ │ └── StatsModel.spec.js ├── unit │ ├── algos │ │ ├── cache │ │ │ ├── GapCache.spec.ts │ │ │ ├── GapSet.spec.ts │ │ │ └── LRUCache.spec.js │ │ ├── heap │ │ │ └── Heap.spec.ts │ │ ├── list │ │ │ ├── MPU.spec.js │ │ │ ├── basic.spec.js │ │ │ ├── delimiter.spec.js │ │ │ ├── delimiterCurrent.spec.js │ │ │ ├── delimiterMaster.spec.ts │ │ │ ├── delimiterNonCurrent.spec.js │ │ │ ├── delimiterOrphanDeleteMarker.spec.js │ │ │ ├── delimiterVersions.spec.js │ │ │ ├── skip.spec.js │ │ │ └── tools.spec.js │ │ ├── set │ │ │ ├── ArrayUtils.spec.js │ │ │ └── SortedSet.spec.js │ │ └── stream │ │ │ └── MergeStream.spec.js │ ├── auth │ │ ├── AuthInfo.spec.js │ │ ├── Vault.spec.js │ │ ├── auth.spec.js │ │ ├── chainBackend.spec.js │ │ ├── in_memory │ │ │ ├── AuthLoader.spec.js │ │ │ ├── backend.spec.js │ │ │ ├── indexer.spec.js │ │ │ ├── sample_authdata.json │ │ │ └── sample_authdata_refresh.json │ │ ├── v2 │ │ │ ├── canonicalization.spec.js │ │ │ ├── checkRequestExpiry.spec.js │ │ │ ├── constructStringToSign.spec.js │ │ │ ├── headerAuthCheck.spec.js │ │ │ ├── publicAccess.spec.js │ │ │ ├── queryAuthCheck.spec.js │ │ │ └── signature.spec.js │ │ └── v4 │ │ │ ├── awsURIencode.spec.js │ │ │ ├── constructStringToSign.spec.js │ │ │ ├── createCanonicalRequest.spec.js │ │ │ ├── generateV4Headers.spec.js │ │ │ ├── headerAuthCheck.spec.js │ │ │ ├── queryAuthCheck.spec.js │ │ │ ├── signingKey.spec.js │ │ │ ├── streamingV4 │ │ │ └── V4Transform.spec.js │ │ │ └── timeUtils.spec.js │ ├── db.spec.js │ ├── errors.spec.ts │ ├── helpers.js │ ├── ipCheck.spec.js │ ├── jsutil.spec.js │ ├── kmip │ │ └── ttlvCodec.spec.js │ ├── matrix.spec.js │ ├── metrics │ │ └── ZenkoMetrics.spec.js │ ├── models │ │ ├── ARN.spec.js │ │ ├── BackendInfo.spec.js │ │ ├── BucketAzureInfo.spec.js │ │ ├── BucketInfo.spec.js │ │ ├── BucketPolicy.spec.js │ │ ├── LifecycleConfiguration.spec.js │ │ ├── NotificationConfiguration.spec.js │ │ ├── ObjectLockConfiguration.spec.js │ │ ├── ObjectMD.spec.js │ │ ├── ObjectMDAmzRestore.spec.js │ │ ├── ObjectMDArchive.spec.js │ │ ├── ObjectMDAzureInfo.spec.js │ │ ├── ObjectMDLocation.spec.js │ │ ├── ReplicationConfiguration.spec.ts │ │ └── WebsiteConfiguration.spec.js │ ├── network │ │ ├── KMSInterface.spec.js │ │ ├── RoundRobin.spec.js │ │ ├── http │ │ │ ├── server.spec.js │ │ │ └── utils.spec.js │ │ ├── probe │ │ │ ├── HealthProbeServer.spec.js │ │ │ ├── ProbeServer.spec.js │ │ │ └── Utils.spec.js │ │ ├── rest │ │ │ ├── index.spec.js │ │ │ └── utils.spec.js │ │ └── rpc │ │ │ ├── level-net.spec.js │ │ │ └── rpc.spec.js │ ├── patches │ │ ├── creds.json │ │ └── locationConstraints.spec.js │ ├── policy │ │ ├── test_policyValidator.spec.js │ │ └── test_principalEvaluator.spec.js │ ├── policyEvaluator.spec.js │ ├── policyEvaluator │ │ ├── RequestContext.spec.js │ │ ├── requestUtils.spec.js │ │ └── utils │ │ │ └── test_checkArnMatch.spec.js │ ├── s3middleware │ │ ├── LifecycleDateTime.spec.js │ │ ├── LifecycleUtils.spec.js │ │ ├── MD5Sum.spec.js │ │ ├── azureHelpers │ │ │ ├── SubStreamingInterface.spec.js │ │ │ └── mpuUtils.spec.js │ │ ├── nullStream.spec.js │ │ ├── objectLegalHold.spec.js │ │ ├── objectRestore.spec.js │ │ ├── objectRetention.spec.js │ │ ├── objectUtils.spec.js │ │ ├── prepareStream.spec.js │ │ ├── processMpuParts.spec.js │ │ ├── tagging.spec.js │ │ ├── userMetadata.spec.js │ │ └── validateConditionalHeaders.spec.js │ ├── s3routes │ │ ├── routeDELETE.spec.js │ │ ├── routeGET.spec.js │ │ ├── routeHEAD.spec.js │ │ ├── routeOPTION.spec.js │ │ ├── routePUT.spec.js │ │ ├── routeWebsite.spec.js │ │ ├── routes.spec.js │ │ └── routesUtils │ │ │ ├── JSONResponseBackend.spec.js │ │ │ ├── XMLResponseBackend.spec.js │ │ │ ├── errorHtmlResponse.spec.js │ │ │ ├── getBucketNameFromHost.spec.js │ │ │ ├── isValidBucketName.spec.js │ │ │ ├── isValidObjectKey.spec.js │ │ │ ├── normalizeRequest.spec.js │ │ │ ├── okHeaderResponse.spec.js │ │ │ ├── redirectRequest.spec.js │ │ │ ├── redirectRequestOnError.spec.js │ │ │ ├── responseBody.spec.ts │ │ │ ├── responseContentHeaders.spec.js │ │ │ ├── responseStreamData.spec.js │ │ │ ├── retrieveData.spec.js │ │ │ └── toArsenalError.spec.ts │ ├── shuffle.spec.ts │ ├── storage │ │ ├── data │ │ │ ├── DataWrapper.spec.js │ │ │ ├── DummyObjectStream.js │ │ │ ├── DummyObjectStream.spec.js │ │ │ ├── DummyService.js │ │ │ ├── external │ │ │ │ ├── AzureClient.spec.js │ │ │ │ ├── ExternalClients.spec.js │ │ │ │ └── GcpService.spec.js │ │ │ ├── in_memory │ │ │ │ └── datastore.spec.js │ │ │ ├── locConstraintParser.spec.js │ │ │ ├── mockClients │ │ │ │ ├── MockAzureClient.js │ │ │ │ └── MockSproxydClient.js │ │ │ └── wrapperClientRoutes.spec.js │ │ ├── metadata │ │ │ ├── bucketclient │ │ │ │ └── LogConsumer.spec.js │ │ │ ├── file │ │ │ │ ├── MetadataFileServer.spec.js │ │ │ │ └── RecordLog.spec.js │ │ │ ├── in_memory │ │ │ │ └── bucket_utilities.spec.js │ │ │ └── mongoclient │ │ │ │ ├── ListRecordStream.spec.js │ │ │ │ ├── LogConsumer.spec.js │ │ │ │ ├── MongoClientInterface.spec.js │ │ │ │ ├── delObject.spec.js │ │ │ │ ├── getObject.spec.js │ │ │ │ ├── getObjects.spec.js │ │ │ │ ├── listObject.spec.js │ │ │ │ ├── putObject.spec.js │ │ │ │ ├── utils.spec.js │ │ │ │ ├── utils │ │ │ │ ├── DummyConfigObject.js │ │ │ │ ├── DummyRequestLogger.js │ │ │ │ └── helper.js │ │ │ │ └── withCond.spec.js │ │ └── utils.spec.js │ ├── stream │ │ └── readJSONStreamObject.spec.js │ ├── stringHash.spec.js │ └── versioning │ │ ├── Version.spec.js │ │ ├── VersionID.spec.js │ │ └── VersioningRequestProcessor.spec.js └── utils │ ├── DummyConfig.js │ ├── DummyKms.js │ ├── DummyRequest.js │ ├── HttpResponseMock.js │ ├── ca.crt │ ├── dummyS3Config.json │ ├── dummyS3ConfigProxy.json │ ├── kmip │ ├── LoopbackServerChannel.js │ ├── badTtlvFixtures.js │ ├── ersatz.js │ ├── lowlevelFixtures.js │ ├── messageFixtures.js │ └── ttlvFixtures.js │ ├── mdProxyUtils.js │ ├── performListing.js │ ├── samplePolicies.json │ ├── test.crt │ └── test.key ├── tsconfig.json └── yarn.lock /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: 2 3 | updates: 4 | - package-ecosystem: npm 5 | directory: "/" 6 | schedule: 7 | interval: daily 8 | time: "13:00" 9 | open-pull-requests-limit: 10 10 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: codeQL 3 | 4 | on: 5 | push: 6 | branches: [development/*, stabilization/*, hotfix/*] 7 | pull_request: 8 | branches: [development/*, stabilization/*, hotfix/*] 9 | workflow_dispatch: 10 | 11 | jobs: 12 | analyze: 13 | name: Static analysis with CodeQL 14 | runs-on: ubuntu-22.04 15 | steps: 16 | - name: Checkout code 17 | uses: actions/checkout@v4 18 | 19 | - name: Initialize CodeQL 20 | uses: github/codeql-action/init@v3 21 | with: 22 | languages: javascript, typescript 23 | 24 | - name: Build and analyze 25 | uses: github/codeql-action/analyze@v3 26 | -------------------------------------------------------------------------------- /.github/workflows/dependency-review.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: dependency review 3 | 4 | on: 5 | pull_request: 6 | branches: [development/*, stabilization/*, hotfix/*] 7 | 8 | jobs: 9 | dependency-review: 10 | runs-on: ubuntu-22.04 11 | steps: 12 | - name: 'Checkout Repository' 13 | uses: actions/checkout@v4 14 | 15 | - name: 'Dependency Review' 16 | uses: actions/dependency-review-action@v4 17 | -------------------------------------------------------------------------------- /.github/workflows/tests.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: tests 3 | 4 | on: 5 | push: 6 | branches-ignore: 7 | - 'development/**' 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-22.04 12 | services: 13 | # Label used to access the service container 14 | redis: 15 | # Docker Hub image 16 | image: redis 17 | # Set health checks to wait until redis has started 18 | options: >- 19 | --health-cmd "redis-cli ping" 20 | --health-interval 10s 21 | --health-timeout 5s 22 | --health-retries 5 23 | ports: 24 | # Maps port 6379 on service container to the host 25 | - 6379:6379 26 | steps: 27 | - name: Checkout 28 | uses: actions/checkout@v4 29 | - uses: actions/setup-node@v4 30 | with: 31 | node-version: '22' 32 | cache: 'yarn' 33 | - name: install dependencies 34 | run: yarn install --frozen-lockfile --prefer-offline --network-concurrency 1 35 | - name: lint yaml 36 | run: yarn --silent lint_yml 37 | - name: lint javascript 38 | run: yarn --silent lint --max-warnings 0 39 | - name: lint markdown 40 | run: yarn --silent lint_md 41 | - name: add hostname 42 | run: | 43 | sudo sh -c "echo '127.0.0.1 testrequestbucket.localhost' >> /etc/hosts" 44 | - name: test and coverage 45 | run: yarn --silent coverage 46 | - name: run functional tests 47 | run: yarn ft_test 48 | - uses: codecov/codecov-action@v4 49 | with: 50 | token: ${{ secrets.CODECOV_TOKEN }} 51 | - name: run executables tests 52 | run: yarn install && yarn test 53 | working-directory: 'lib/executables/pensieveCreds/' 54 | 55 | compile: 56 | name: Compile and upload build artifacts 57 | needs: test 58 | runs-on: ubuntu-22.04 59 | steps: 60 | - name: Checkout 61 | uses: actions/checkout@v4 62 | - name: Install NodeJS 63 | uses: actions/setup-node@v4 64 | with: 65 | node-version: '22' 66 | cache: yarn 67 | - name: Install dependencies 68 | run: yarn install --frozen-lockfile --prefer-offline 69 | - name: Compile 70 | run: yarn build 71 | - name: Upload artifacts 72 | uses: scality/action-artifacts@v4 73 | with: 74 | url: https://artifacts.scality.net 75 | user: ${{ secrets.ARTIFACTS_USER }} 76 | password: ${{ secrets.ARTIFACTS_PASSWORD }} 77 | source: ./build 78 | method: upload 79 | if: success() 80 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | *.log 3 | 4 | # Dependency directory 5 | node_modules/ 6 | */node_modules/ 7 | 8 | # Build executables 9 | *-win.exe 10 | *-linux 11 | *-macos 12 | 13 | # Coverage 14 | coverage/ 15 | .nyc_output/ 16 | 17 | # TypeScript 18 | build/ 19 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scality/Arsenal/fc9057ee107315601e9c1b17dfddbd16ab5859cf/.npmignore -------------------------------------------------------------------------------- /.yamllint: -------------------------------------------------------------------------------- 1 | extends: default 2 | rules: 3 | document-start: {level: error} 4 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing rules 2 | 3 | Please follow the 4 | [Contributing Guidelines]( 5 | https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md). 6 | -------------------------------------------------------------------------------- /babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [ 3 | ['@babel/preset-env', { targets: { node: 'current' } }], 4 | '@babel/preset-typescript', 5 | ], 6 | }; 7 | -------------------------------------------------------------------------------- /circle.yml: -------------------------------------------------------------------------------- 1 | --- 2 | general: 3 | branches: 4 | ignore: 5 | - /^ultron\/.*/ # Ignore ultron/* branches 6 | 7 | machine: 8 | node: 9 | version: 6.13.1 10 | services: 11 | - redis 12 | environment: 13 | CXX: g++-4.9 14 | 15 | dependencies: 16 | override: 17 | - rm -rf node_modules 18 | - npm install 19 | - sudo pip install yamllint 20 | 21 | test: 22 | override: 23 | - npm run --silent lint_yml 24 | - npm run --silent lint -- --max-warnings 0 25 | - npm run --silent lint_md 26 | - npm run --silent test 27 | - npm run ft_test 28 | - cd lib/executables/pensieveCreds && npm install && npm test 29 | -------------------------------------------------------------------------------- /documentation/listingAlgos/delimiter.md: -------------------------------------------------------------------------------- 1 | # Delimiter 2 | 3 | The Delimiter class handles raw listings from the database with an 4 | optional delimiter, and fills in a curated listing with "Contents" and 5 | "CommonPrefixes" as a result. 6 | 7 | ## Expected Behavior 8 | 9 | - only lists keys belonging to the given **prefix** (if provided) 10 | 11 | - groups listed keys that have a common prefix ending with a delimiter 12 | inside CommonPrefixes 13 | 14 | - can take a **marker** or **continuationToken** to list from a specific key 15 | 16 | - can take a **maxKeys** parameter to limit how many keys can be returned 17 | 18 | ## State Chart 19 | 20 | - States with grey background are *Idle* states, which are waiting for 21 | a new listing key 22 | 23 | - States with blue background are *Processing* states, which are 24 | actively processing a new listing key passed by the filter() 25 | function 26 | 27 | ![Delimiter State Chart](./pics/delimiterStateChart.svg) 28 | -------------------------------------------------------------------------------- /documentation/listingAlgos/delimiterMaster.md: -------------------------------------------------------------------------------- 1 | # DelimiterMaster 2 | 3 | The DelimiterMaster class handles raw listings from the database of a 4 | versioned or non-versioned bucket with an optional delimiter, and 5 | fills in a curated listing with "Contents" and "CommonPrefixes" as a 6 | result. 7 | 8 | ## Expected Behavior 9 | 10 | - only lists latest versions of versioned buckets 11 | 12 | - only lists keys belonging to the given **prefix** (if provided) 13 | 14 | - does not list latest versions that are delete markers 15 | 16 | - groups listed keys that have a common prefix ending with a delimiter 17 | inside CommonPrefixes 18 | 19 | - can take a **marker** or **continuationToken** to list from a specific key 20 | 21 | - can take a **maxKeys** parameter to limit how many keys can be returned 22 | 23 | - reconciles internal PHD keys with the next version (those are 24 | created when a specific version that is the latest version is 25 | deleted) 26 | 27 | - skips internal keys like replay keys 28 | 29 | ## State Chart 30 | 31 | - States with grey background are *Idle* states, which are waiting for 32 | a new listing key 33 | 34 | - States with blue background are *Processing* states, which are 35 | actively processing a new listing key passed by the filter() 36 | function 37 | 38 | ### Bucket Vformat=v0 39 | 40 | ![DelimiterMaster State Chart for v0 format](./pics/delimiterMasterV0StateChart.svg) 41 | 42 | ### Bucket Vformat=v1 43 | 44 | For buckets in versioning key format **v1**, the algorithm used is the 45 | one from [Delimiter](delimiter.md). 46 | -------------------------------------------------------------------------------- /documentation/listingAlgos/delimiterVersions.md: -------------------------------------------------------------------------------- 1 | # DelimiterVersions 2 | 3 | The DelimiterVersions class handles raw listings from the database of a 4 | versioned or non-versioned bucket with an optional delimiter, and 5 | fills in a curated listing with "Versions" and "CommonPrefixes" as a 6 | result. 7 | 8 | ## Expected Behavior 9 | 10 | - lists individual distinct versions of versioned buckets 11 | 12 | - only lists keys belonging to the given **prefix** (if provided) 13 | 14 | - groups listed keys that have a common prefix ending with a delimiter 15 | inside CommonPrefixes 16 | 17 | - can take a **keyMarker** and optionally a **versionIdMarker** to 18 | list from a specific key or version 19 | 20 | - can take a **maxKeys** parameter to limit how many keys can be returned 21 | 22 | - skips internal keys like replay keys 23 | 24 | ## State Chart 25 | 26 | - States with grey background are *Idle* states, which are waiting for 27 | a new listing key 28 | 29 | - States with blue background are *Processing* states, which are 30 | actively processing a new listing key passed by the filter() 31 | function 32 | 33 | ![DelimiterVersions State Chart](./pics/delimiterVersionsStateChart.svg) 34 | -------------------------------------------------------------------------------- /documentation/listingAlgos/pics/delimiterStateChart.dot: -------------------------------------------------------------------------------- 1 | digraph { 2 | node [shape="box",style="filled,rounded",fontsize=16,fixedsize=true,width=3]; 3 | edge [fontsize=14]; 4 | rankdir=TB; 5 | 6 | START [shape="circle",width=0.2,label="",style="filled",fillcolor="black"] 7 | END [shape="circle",width=0.2,label="",style="filled",fillcolor="black",peripheries=2] 8 | 9 | node [fillcolor="lightgrey"]; 10 | "NotSkipping.Idle" [label="NotSkipping",group="NotSkipping"]; 11 | "NeverSkipping.Idle" [label="NeverSkipping",group="NeverSkipping"]; 12 | "NotSkippingPrefix.Idle" [label="NotSkippingPrefix",group="NotSkippingPrefix"]; 13 | "SkippingPrefix.Idle" [label="SkippingPrefix",group="SkippingPrefix"]; 14 | 15 | node [fillcolor="lightblue"]; 16 | "NeverSkipping.Processing" [label="NeverSkipping",group="NeverSkipping"]; 17 | "NotSkippingPrefix.Processing" [label="NotSkippingPrefix",group="NotSkippingPrefix"]; 18 | "SkippingPrefix.Processing" [label="SkippingPrefix",group="SkippingPrefix"]; 19 | 20 | START -> "NotSkipping.Idle" 21 | "NotSkipping.Idle" -> "NeverSkipping.Idle" [label="[delimiter == undefined]"] 22 | "NotSkipping.Idle" -> "NotSkippingPrefix.Idle" [label="[delimiter == '/']"] 23 | 24 | "NeverSkipping.Idle" -> "NeverSkipping.Processing" [label="filter(key, value)"] 25 | "NotSkippingPrefix.Idle" -> "NotSkippingPrefix.Processing" [label="filter(key, value)"] 26 | "SkippingPrefix.Idle" -> "SkippingPrefix.Processing" [label="filter(key, value)"] 27 | 28 | "NeverSkipping.Processing" -> END [label="[nKeys == maxKeys]\n-> FILTER_END"] 29 | "NeverSkipping.Processing" -> "NeverSkipping.Idle" [label="[nKeys < maxKeys]\n/ Contents.append(key, value)\n -> FILTER_ACCEPT"] 30 | "NotSkippingPrefix.Processing" -> END [label="[nKeys == maxKeys]\n -> FILTER_END"] 31 | "NotSkippingPrefix.Processing" -> "SkippingPrefix.Idle" [label="[nKeys < maxKeys and hasDelimiter(key)]\n/ prefix <- prefixOf(key)\n/ CommonPrefixes.append(prefixOf(key))\n-> FILTER_ACCEPT"] 32 | "NotSkippingPrefix.Processing" -> "NotSkippingPrefix.Idle" [label="[nKeys < maxKeys and not hasDelimiter(key)]\n/ Contents.append(key, value)\n -> FILTER_ACCEPT"] 33 | "SkippingPrefix.Processing" -> "SkippingPrefix.Idle" [label="[key.startsWith(prefix)]\n-> FILTER_SKIP"] 34 | "SkippingPrefix.Processing" -> "NotSkippingPrefix.Processing" [label="[not key.startsWith(prefix)]"] 35 | } 36 | -------------------------------------------------------------------------------- /eslint.config.mjs: -------------------------------------------------------------------------------- 1 | import path from "node:path"; 2 | import { fileURLToPath } from "node:url"; 3 | import js from "@eslint/js"; 4 | import { FlatCompat } from "@eslint/eslintrc"; 5 | import tseslint from 'typescript-eslint'; 6 | import { includeIgnoreFile } from '@eslint/compat'; 7 | 8 | 9 | const __filename = fileURLToPath(import.meta.url); 10 | const __dirname = path.dirname(__filename); 11 | const gitignorePath = path.resolve(__dirname, '.gitignore'); 12 | const compat = new FlatCompat({ 13 | baseDirectory: __dirname, 14 | recommendedConfig: js.configs.recommended, 15 | allConfig: js.configs.all 16 | }); 17 | 18 | export default tseslint.config( 19 | ...compat.extends('@scality/scality'), 20 | ...tseslint.configs.recommended, 21 | includeIgnoreFile(gitignorePath), 22 | { 23 | rules: { 24 | '@typescript-eslint/ban-ts-comment': 'off', 25 | '@typescript-eslint/no-require-imports': 'off', 26 | '@typescript-eslint/no-explicit-any': 'off', 27 | '@typescript-eslint/no-empty-object-type': 'off', 28 | 'camelcase': 'off', 29 | 'no-param-reassign': 'off', 30 | 'new-cap': 'off', 31 | 'quotes': 'off', 32 | '@typescript-eslint/no-unsafe-function-type':'off' 33 | } 34 | } 35 | ); 36 | -------------------------------------------------------------------------------- /greenkeeper.json: -------------------------------------------------------------------------------- 1 | { 2 | "groups": { 3 | "default": { 4 | "packages": [ 5 | "lib/executables/pensieveCreds/package.json", 6 | "package.json" 7 | ] 8 | } 9 | }, 10 | "branchPrefix": "improvement/greenkeeper.io/", 11 | "commitMessages": { 12 | "initialBadge": "docs(readme): add Greenkeeper badge", 13 | "initialDependencies": "chore(package): update dependencies", 14 | "initialBranches": "chore(bert-e): whitelist greenkeeper branches", 15 | "dependencyUpdate": "fix(package): update ${dependency} to version ${version}", 16 | "devDependencyUpdate": "chore(package): update ${dependency} to version ${version}", 17 | "dependencyPin": "fix: pin ${dependency} to ${oldVersionResolved}", 18 | "devDependencyPin": "chore: pin ${dependency} to ${oldVersionResolved}", 19 | "closes": "\n\nCloses #${number}" 20 | }, 21 | "ignore": [ 22 | "ajv", 23 | "eslint", 24 | "eslint-plugin-react", 25 | "eslint-config-airbnb", 26 | "eslint-config-scality" 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /lib/algos/list/exportAlgos.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | Basic: require('./basic').List, 3 | Delimiter: require('./delimiter').Delimiter, 4 | DelimiterVersions: require('./delimiterVersions') 5 | .DelimiterVersions, 6 | DelimiterMaster: require('./delimiterMaster') 7 | .DelimiterMaster, 8 | MPU: require('./MPU').MultipartUploads, 9 | DelimiterCurrent: require('./delimiterCurrent').DelimiterCurrent, 10 | DelimiterNonCurrent: require('./delimiterNonCurrent').DelimiterNonCurrent, 11 | DelimiterOrphanDeleteMarker: require('./delimiterOrphanDeleteMarker').DelimiterOrphanDeleteMarker, 12 | }; 13 | -------------------------------------------------------------------------------- /lib/algos/list/skip.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | 3 | const { FILTER_END, FILTER_SKIP, SKIP_NONE } = require('./tools'); 4 | 5 | 6 | const MAX_STREAK_LENGTH = 100; 7 | 8 | /** 9 | * Handle the filtering and the skip mechanism of a listing result. 10 | */ 11 | class Skip { 12 | /** 13 | * @param {Object} params - skip parameters 14 | * @param {Object} params.extension - delimiter extension used (required) 15 | * @param {String | string[]} params.gte - current range gte (greater than or 16 | * equal) used by the client code 17 | */ 18 | constructor(params) { 19 | assert(params.extension); 20 | 21 | this.extension = params.extension; 22 | this.gteParams = params.gte; 23 | 24 | this.listingEndCb = null; 25 | this.skipRangeCb = null; 26 | 27 | /* Used to count consecutive FILTER_SKIP returned by the extension 28 | * filter method. Once this counter reaches MAX_STREAK_LENGTH, the 29 | * filter function tries to skip unwanted values by defining a new 30 | * range. */ 31 | this.streakLength = 0; 32 | } 33 | 34 | setListingEndCb(cb) { 35 | this.listingEndCb = cb; 36 | } 37 | 38 | setSkipRangeCb(cb) { 39 | this.skipRangeCb = cb; 40 | } 41 | 42 | /** 43 | * Filter an entry. 44 | * @param {Object} entry - entry to filter. 45 | * @return {undefined} 46 | * 47 | * This function calls the listing end or the skip range callbacks if 48 | * needed. 49 | */ 50 | filter(entry) { 51 | assert(this.listingEndCb); 52 | assert(this.skipRangeCb); 53 | 54 | const filteringResult = this.extension.filter(entry); 55 | const skipTo = this.extension.skipping(); 56 | 57 | if (filteringResult === FILTER_END) { 58 | this.listingEndCb(); 59 | } else if (filteringResult === FILTER_SKIP 60 | && skipTo !== SKIP_NONE) { 61 | if (++this.streakLength >= MAX_STREAK_LENGTH) { 62 | let newRange; 63 | if (Array.isArray(skipTo)) { 64 | newRange = []; 65 | for (let i = 0; i < skipTo.length; ++i) { 66 | newRange.push(skipTo[i]); 67 | } 68 | } else { 69 | newRange = skipTo; 70 | } 71 | /* Avoid to loop on the same range again and again. */ 72 | if (newRange === this.gteParams) { 73 | this.streakLength = 1; 74 | } else { 75 | this.skipRangeCb(newRange); 76 | } 77 | } 78 | } else { 79 | this.streakLength = 0; 80 | } 81 | } 82 | } 83 | 84 | 85 | module.exports = Skip; 86 | -------------------------------------------------------------------------------- /lib/algos/list/tools.js: -------------------------------------------------------------------------------- 1 | const { DbPrefixes } = require('../../versioning/constants').VersioningConstants; 2 | 3 | // constants for extensions 4 | const SKIP_NONE = undefined; // to be inline with the values of NextMarker 5 | const FILTER_ACCEPT = 1; 6 | const FILTER_SKIP = 0; 7 | const FILTER_END = -1; 8 | 9 | /** 10 | * This function check if number is valid 11 | * To be valid a number need to be an Integer and be lower than the limit 12 | * if specified 13 | * If the number is not valid the limit is returned 14 | * @param {Number} number - The number to check 15 | * @param {Number} limit - The limit to respect 16 | * @return {Number} - The parsed number || limit 17 | */ 18 | function checkLimit(number, limit) { 19 | const parsed = Number.parseInt(number, 10); 20 | const valid = !Number.isNaN(parsed) && (!limit || parsed <= limit); 21 | return valid ? parsed : limit; 22 | } 23 | 24 | /** 25 | * Increment the charCode of the last character of a valid string. 26 | * 27 | * @param {string} str - the input string 28 | * @return {string} - the incremented string 29 | * or the input if it is not valid 30 | */ 31 | function inc(str) { 32 | return str ? (str.slice(0, str.length - 1) + 33 | String.fromCharCode(str.charCodeAt(str.length - 1) + 1)) : str; 34 | } 35 | 36 | /** 37 | * Transform listing parameters for v0 versioning key format to make 38 | * it compatible with v1 format 39 | * 40 | * @param {object} v0params - listing parameters for v0 format 41 | * @return {object} - listing parameters for v1 format 42 | */ 43 | function listingParamsMasterKeysV0ToV1(v0params) { 44 | const v1params = Object.assign({}, v0params); 45 | if (v0params.gt !== undefined) { 46 | v1params.gt = `${DbPrefixes.Master}${v0params.gt}`; 47 | } else if (v0params.gte !== undefined) { 48 | v1params.gte = `${DbPrefixes.Master}${v0params.gte}`; 49 | } else { 50 | v1params.gte = DbPrefixes.Master; 51 | } 52 | if (v0params.lt !== undefined) { 53 | v1params.lt = `${DbPrefixes.Master}${v0params.lt}`; 54 | } else if (v0params.lte !== undefined) { 55 | v1params.lte = `${DbPrefixes.Master}${v0params.lte}`; 56 | } else { 57 | v1params.lt = inc(DbPrefixes.Master); // stop after the last master key 58 | } 59 | return v1params; 60 | } 61 | 62 | module.exports = { 63 | checkLimit, 64 | inc, 65 | listingParamsMasterKeysV0ToV1, 66 | SKIP_NONE, 67 | FILTER_END, 68 | FILTER_SKIP, 69 | FILTER_ACCEPT, 70 | }; 71 | -------------------------------------------------------------------------------- /lib/algos/set/ArrayUtils.js: -------------------------------------------------------------------------------- 1 | function indexOf(arr, value) { 2 | if (!arr.length) { 3 | return -1; 4 | } 5 | let lo = 0; 6 | let hi = arr.length - 1; 7 | 8 | while (hi - lo > 1) { 9 | const i = lo + ((hi - lo) >> 1); 10 | if (arr[i] > value) { 11 | hi = i; 12 | } else { 13 | lo = i; 14 | } 15 | } 16 | if (arr[lo] === value) { 17 | return lo; 18 | } 19 | if (arr[hi] === value) { 20 | return hi; 21 | } 22 | return -1; 23 | } 24 | 25 | function indexAtOrBelow(arr, value) { 26 | let i; 27 | let lo; 28 | let hi; 29 | 30 | if (!arr.length || arr[0] > value) { 31 | return -1; 32 | } 33 | if (arr[arr.length - 1] <= value) { 34 | return arr.length - 1; 35 | } 36 | 37 | lo = 0; 38 | hi = arr.length - 1; 39 | 40 | while (hi - lo > 1) { 41 | i = lo + ((hi - lo) >> 1); 42 | if (arr[i] > value) { 43 | hi = i; 44 | } else { 45 | lo = i; 46 | } 47 | } 48 | 49 | return lo; 50 | } 51 | 52 | /* 53 | * perform symmetric diff in O(m + n) 54 | */ 55 | function symDiff(k1, k2, v1, v2, cb) { 56 | let i = 0; 57 | let j = 0; 58 | const n = k1.length; 59 | const m = k2.length; 60 | 61 | while (i < n && j < m) { 62 | if (k1[i] < k2[j]) { 63 | cb(v1[i]); 64 | i++; 65 | } else if (k2[j] < k1[i]) { 66 | cb(v2[j]); 67 | j++; 68 | } else { 69 | i++; 70 | j++; 71 | } 72 | } 73 | while (i < n) { 74 | cb(v1[i]); 75 | i++; 76 | } 77 | while (j < m) { 78 | cb(v2[j]); 79 | j++; 80 | } 81 | } 82 | 83 | module.exports = { 84 | indexOf, 85 | indexAtOrBelow, 86 | symDiff, 87 | }; 88 | -------------------------------------------------------------------------------- /lib/algos/set/SortedSet.js: -------------------------------------------------------------------------------- 1 | const ArrayUtils = require('./ArrayUtils'); 2 | 3 | class SortedSet { 4 | constructor(obj) { 5 | if (obj) { 6 | this.keys = obj.keys; 7 | this.values = obj.values; 8 | } else { 9 | this.clear(); 10 | } 11 | } 12 | 13 | clear() { 14 | this.keys = []; 15 | this.values = []; 16 | } 17 | 18 | get size() { 19 | return this.keys.length; 20 | } 21 | 22 | set(key, value) { 23 | const index = ArrayUtils.indexAtOrBelow(this.keys, key); 24 | if (this.keys[index] === key) { 25 | this.values[index] = value; 26 | return; 27 | } 28 | this.keys.splice(index + 1, 0, key); 29 | this.values.splice(index + 1, 0, value); 30 | } 31 | 32 | isSet(key) { 33 | const index = ArrayUtils.indexOf(this.keys, key); 34 | return index >= 0; 35 | } 36 | 37 | get(key) { 38 | const index = ArrayUtils.indexOf(this.keys, key); 39 | return index >= 0 ? this.values[index] : undefined; 40 | } 41 | 42 | del(key) { 43 | const index = ArrayUtils.indexOf(this.keys, key); 44 | if (index >= 0) { 45 | this.keys.splice(index, 1); 46 | this.values.splice(index, 1); 47 | } 48 | } 49 | } 50 | 51 | module.exports = SortedSet; 52 | -------------------------------------------------------------------------------- /lib/auth/backends/in_memory/types.ts: -------------------------------------------------------------------------------- 1 | import joi from 'joi'; 2 | 3 | export type Callback = (err?: Error | null | undefined, data?: Data) => void; 4 | 5 | export type Credentials = { access: string; secret: string }; 6 | export type Base = { 7 | arn: string; 8 | canonicalID: string; 9 | shortid: string; 10 | email: string; 11 | keys: Credentials[]; 12 | }; 13 | export type Account = Base & { name: string; users: any[] }; 14 | export type Accounts = { accounts: Account[] }; 15 | export type Entity = Base & { accountDisplayName: string }; 16 | 17 | const keys = ((): joi.ArraySchema => { 18 | const str = joi.string().required(); 19 | const items = { access: str, secret: str }; 20 | return joi.array().items(items).required(); 21 | })(); 22 | 23 | const account = (() => joi.object({ 24 | name: joi.string().required(), 25 | email: joi.string().email().required(), 26 | arn: joi.string().required(), 27 | canonicalID: joi.string().required(), 28 | shortid: joi 29 | .string() 30 | .regex(/^[0-9]{12}$/) 31 | .required(), 32 | keys, 33 | // backward-compat 34 | users: joi.array(), 35 | }))(); 36 | 37 | const accounts = (() => joi.object({ 38 | accounts: joi 39 | .array() 40 | .items(account) 41 | .required() 42 | .unique('arn') 43 | .unique('email') 44 | .unique('canonicalID'), 45 | }))(); 46 | 47 | export const validators = { keys, account, accounts }; 48 | -------------------------------------------------------------------------------- /lib/auth/backends/in_memory/validateAuthConfig.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from 'werelogs'; 2 | import AuthLoader from './AuthLoader'; 3 | import { Accounts } from './types'; 4 | 5 | /** 6 | * @deprecated please use {@link AuthLoader} class instead 7 | * @return true on erroneous data false on success 8 | */ 9 | export default function validateAuthConfig( 10 | authdata: Accounts, 11 | logApi?: { Logger: typeof Logger } 12 | ) { 13 | const authLoader = new AuthLoader(logApi); 14 | authLoader.addAccounts(authdata); 15 | return !authLoader.validate(); 16 | } 17 | -------------------------------------------------------------------------------- /lib/auth/backends/in_memory/vaultUtilities.ts: -------------------------------------------------------------------------------- 1 | import * as crypto from 'crypto'; 2 | 3 | /** hashSignature for v2 Auth 4 | * @param {string} stringToSign - built string to sign per AWS rules 5 | * @param {string} secretKey - user's secretKey 6 | * @param {string} algorithm - either SHA256 or SHA1 7 | * @return {string} reconstructed signature 8 | */ 9 | export function hashSignature( 10 | stringToSign: string, 11 | secretKey: string, 12 | algorithm: 'SHA256' | 'SHA1' 13 | ): string { 14 | const hmacObject = crypto.createHmac(algorithm, secretKey); 15 | return hmacObject.update(stringToSign, 'binary').digest('base64'); 16 | } 17 | 18 | const sha256Digest = (key: string | Buffer, data: string) => 19 | crypto.createHmac('sha256', key).update(data, 'binary').digest(); 20 | 21 | /** calculateSigningKey for v4 Auth 22 | * @param {string} secretKey - requester's secretKey 23 | * @param {string} region - region included in request 24 | * @param {string} scopeDate - scopeDate included in request 25 | * @param {string} [service] - To specify another service than s3 26 | * @return {string} signingKey - signingKey to calculate signature 27 | */ 28 | export function calculateSigningKey( 29 | secretKey: string, 30 | region: string, 31 | scopeDate: string, 32 | service?: string 33 | ): Buffer { 34 | const dateKey = sha256Digest(`AWS4${secretKey}`, scopeDate); 35 | const dateRegionKey = sha256Digest(dateKey, region); 36 | const dateRegionServiceKey = sha256Digest(dateRegionKey, service || 's3'); 37 | const signingKey = sha256Digest(dateRegionServiceKey, 'aws4_request'); 38 | return signingKey; 39 | } 40 | -------------------------------------------------------------------------------- /lib/auth/v2/algoCheck.ts: -------------------------------------------------------------------------------- 1 | export default function algoCheck(signatureLength: number) { 2 | let algo: 'sha256' | 'sha1'; 3 | // If the signature sent is 44 characters, 4 | // this means that sha256 was used: 5 | // 44 characters in base64 6 | const SHA256LEN = 44; 7 | const SHA1LEN = 28; 8 | if (signatureLength === SHA256LEN) { 9 | algo = 'sha256'; 10 | } 11 | if (signatureLength === SHA1LEN) { 12 | algo = 'sha1'; 13 | } 14 | // @ts-ignore 15 | return algo; 16 | } 17 | -------------------------------------------------------------------------------- /lib/auth/v2/authV2.ts: -------------------------------------------------------------------------------- 1 | export * as header from './headerAuthCheck'; 2 | export * as query from './queryAuthCheck'; 3 | -------------------------------------------------------------------------------- /lib/auth/v2/checkRequestExpiry.ts: -------------------------------------------------------------------------------- 1 | import type { RequestLogger } from 'werelogs'; 2 | import errors from '../../errors'; 3 | 4 | const epochTime = new Date('1970-01-01').getTime(); 5 | 6 | export default function checkRequestExpiry(timestamp: number, log: RequestLogger) { 7 | // If timestamp is before epochTime, the request is invalid and return 8 | // errors.AccessDenied 9 | if (timestamp < epochTime) { 10 | log.debug('request time is invalid', { timestamp }); 11 | return errors.AccessDenied; 12 | } 13 | // If timestamp is not within 15 minutes of current time, or if 14 | // timestamp is more than 15 minutes in the future, the request 15 | // has expired and return errors.RequestTimeTooSkewed 16 | const currentTime = Date.now(); 17 | log.trace('request timestamp', { requestTimestamp: timestamp }); 18 | log.trace('current timestamp', { currentTimestamp: currentTime }); 19 | 20 | const fifteenMinutes = (15 * 60 * 1000); 21 | if (currentTime - timestamp > fifteenMinutes) { 22 | log.trace('request timestamp is not within 15 minutes of current time'); 23 | log.debug('request time too skewed', { timestamp }); 24 | return errors.RequestTimeTooSkewed; 25 | } 26 | 27 | if (currentTime + fifteenMinutes < timestamp) { 28 | log.trace('request timestamp is more than 15 minutes into future'); 29 | log.debug('request time too skewed', { timestamp }); 30 | return errors.RequestTimeTooSkewed; 31 | } 32 | 33 | return undefined; 34 | } 35 | -------------------------------------------------------------------------------- /lib/auth/v2/constructStringToSign.ts: -------------------------------------------------------------------------------- 1 | import type { RequestLogger } from 'werelogs'; 2 | import utf8 from 'utf8'; 3 | import getCanonicalizedAmzHeaders from './getCanonicalizedAmzHeaders'; 4 | import getCanonicalizedResource from './getCanonicalizedResource'; 5 | 6 | export default function constructStringToSign( 7 | request: any, 8 | data: Record, 9 | log: RequestLogger, 10 | clientType?: any, 11 | ) { 12 | /* 13 | Build signature per AWS requirements: 14 | StringToSign = HTTP-Verb + '\n' + 15 | Content-MD5 + '\n' + 16 | Content-Type + '\n' + 17 | Date (or Expiration for query Auth) + '\n' + 18 | CanonicalizedAmzHeaders + 19 | CanonicalizedResource; 20 | */ 21 | log.trace('constructing string to sign'); 22 | 23 | let stringToSign = `${request.method}\n`; 24 | const headers = request.headers; 25 | const query = data; 26 | 27 | const contentMD5 = headers['content-md5'] ? 28 | headers['content-md5'] : query['Content-MD5']; 29 | stringToSign += (contentMD5 ? `${contentMD5}\n` : '\n'); 30 | 31 | const contentType = headers['content-type'] ? 32 | headers['content-type'] : query['Content-Type']; 33 | stringToSign += (contentType ? `${contentType}\n` : '\n'); 34 | 35 | /* 36 | AWS docs are conflicting on whether to include x-amz-date header here 37 | if present in request. 38 | s3cmd includes x-amz-date in amzHeaders rather 39 | than here in stringToSign so we have replicated that. 40 | */ 41 | const date = query.Expires ? query.Expires : headers.date; 42 | const combinedQueryHeaders = Object.assign({}, headers, query); 43 | stringToSign += (date ? `${date}\n` : '\n') 44 | + getCanonicalizedAmzHeaders(combinedQueryHeaders, clientType) 45 | + getCanonicalizedResource(request, clientType); 46 | return utf8.encode(stringToSign); 47 | } 48 | -------------------------------------------------------------------------------- /lib/auth/v2/getCanonicalizedAmzHeaders.ts: -------------------------------------------------------------------------------- 1 | export default function getCanonicalizedAmzHeaders(headers: Headers, clientType: string) { 2 | /* 3 | Iterate through headers and pull any headers that are x-amz headers. 4 | Need to include 'x-amz-date' here even though AWS docs 5 | ambiguous on this. 6 | */ 7 | const filterFn = clientType === 'GCP' ? 8 | (val: string) => val.substr(0, 7) === 'x-goog-' : 9 | (val: string) => val.substr(0, 6) === 'x-amz-'; 10 | const amzHeaders = Object.keys(headers) 11 | .filter(filterFn) 12 | .map(val => [val.trim(), headers[val].trim()]); 13 | /* 14 | AWS docs state that duplicate headers should be combined 15 | in the same header with values concatenated with 16 | a comma separation. 17 | Node combines duplicate headers and concatenates the values 18 | with a comma AND SPACE separation. 19 | Could replace all occurrences of ', ' with ',' but this 20 | would remove spaces that might be desired 21 | (for instance, in date header). 22 | Opted to proceed without this parsing since it does not appear 23 | that the AWS clients use duplicate headers. 24 | */ 25 | 26 | // If there are no amz headers, just return an empty string 27 | if (amzHeaders.length === 0) { 28 | return ''; 29 | } 30 | 31 | 32 | // Sort the amz headers by key (first item in tuple) 33 | amzHeaders.sort((a, b) => { 34 | if (a[0] > b[0]) { 35 | return 1; 36 | } 37 | return -1; 38 | }); 39 | // Build headerString 40 | return amzHeaders.reduce((headerStr, current) => 41 | `${headerStr}${current[0]}:${current[1]}\n`, 42 | ''); 43 | } 44 | -------------------------------------------------------------------------------- /lib/auth/v4/authV4.ts: -------------------------------------------------------------------------------- 1 | export * as header from './headerAuthCheck'; 2 | export * as query from './queryAuthCheck'; 3 | -------------------------------------------------------------------------------- /lib/auth/v4/constructStringToSign.ts: -------------------------------------------------------------------------------- 1 | import * as crypto from 'crypto'; 2 | import type { RequestLogger } from 'werelogs'; 3 | import createCanonicalRequest from './createCanonicalRequest'; 4 | 5 | /** 6 | * constructStringToSign - creates V4 stringToSign 7 | * @param {object} params - params object 8 | * @returns {string} - stringToSign 9 | */ 10 | export default function constructStringToSign(params: { 11 | request: any; 12 | signedHeaders: any; 13 | payloadChecksum: any; 14 | credentialScope: string; 15 | timestamp: string; 16 | query: Record; 17 | log?: RequestLogger; 18 | proxyPath?: string; 19 | awsService: string; 20 | }): string { 21 | const { 22 | request, 23 | signedHeaders, 24 | payloadChecksum, 25 | credentialScope, 26 | timestamp, 27 | query, 28 | log, 29 | proxyPath, 30 | } = params; 31 | const path = proxyPath || request.path; 32 | 33 | const canonicalReqResult = createCanonicalRequest({ 34 | pHttpVerb: request.method, 35 | pResource: path, 36 | pQuery: query, 37 | pHeaders: request.headers, 38 | pSignedHeaders: signedHeaders, 39 | payloadChecksum, 40 | service: params.awsService, 41 | }); 42 | 43 | if (log) { 44 | log.debug('constructed canonicalRequest', { canonicalReqResult }); 45 | } 46 | const sha256 = crypto.createHash('sha256'); 47 | const canonicalHex = sha256.update(canonicalReqResult, 'binary') 48 | .digest('hex'); 49 | const stringToSign = `AWS4-HMAC-SHA256\n${timestamp}\n` + 50 | `${credentialScope}\n${canonicalHex}`; 51 | return stringToSign; 52 | } 53 | -------------------------------------------------------------------------------- /lib/auth/v4/streamingV4/constructChunkStringToSign.ts: -------------------------------------------------------------------------------- 1 | import * as crypto from 'crypto'; 2 | import * as constants from '../../../constants'; 3 | 4 | /** 5 | * Constructs stringToSign for chunk 6 | * @param timestamp - date parsed from headers 7 | * in ISO 8601 format: YYYYMMDDTHHMMSSZ 8 | * @param credentialScope - items from auth 9 | * header plus the string 'aws4_request' joined with '/': 10 | * timestamp/region/aws-service/aws4_request 11 | * @param lastSignature - signature from headers or prior chunk 12 | * @param justDataChunk - data portion of chunk 13 | * @returns stringToSign 14 | */ 15 | export default function constructChunkStringToSign( 16 | timestamp: string, 17 | credentialScope: string, 18 | lastSignature: string, 19 | justDataChunk?: Buffer | string, 20 | ) { 21 | let currentChunkHash: string; 22 | // for last chunk, there will be no data, so use emptyStringHash 23 | if (!justDataChunk) { 24 | currentChunkHash = constants.emptyStringHash; 25 | } else { 26 | const hash = crypto.createHash('sha256'); 27 | const temp = justDataChunk instanceof Buffer 28 | ? hash.update(justDataChunk) 29 | : hash.update(justDataChunk, 'binary'); 30 | currentChunkHash = temp.digest('hex'); 31 | } 32 | return `AWS4-HMAC-SHA256-PAYLOAD\n${timestamp}\n` + 33 | `${credentialScope}\n${lastSignature}\n` + 34 | `${constants.emptyStringHash}\n${currentChunkHash}`; 35 | } 36 | -------------------------------------------------------------------------------- /lib/auth/v4/timeUtils.ts: -------------------------------------------------------------------------------- 1 | import type { RequestLogger } from 'werelogs'; 2 | 3 | /** 4 | * Convert timestamp to milliseconds since Unix Epoch 5 | * @param timestamp of ISO8601Timestamp format without 6 | * dashes or colons, e.g. 20160202T220410Z 7 | */ 8 | export function convertAmzTimeToMs(timestamp: string) { 9 | const arr = timestamp.split(''); 10 | // Convert to YYYY-MM-DDTHH:mm:ss.sssZ 11 | const ISO8601time = `${arr.slice(0, 4).join('')}-${arr[4]}${arr[5]}` + 12 | `-${arr.slice(6, 11).join('')}:${arr[11]}${arr[12]}:${arr[13]}` + 13 | `${arr[14]}.000Z`; 14 | return Date.parse(ISO8601time); 15 | } 16 | 17 | /** 18 | * Convert UTC timestamp to ISO 8601 timestamp 19 | * @param timestamp of UTC form: Fri, 10 Feb 2012 21:34:55 GMT 20 | * @return ISO8601 timestamp of form: YYYYMMDDTHHMMSSZ 21 | */ 22 | export function convertUTCtoISO8601(timestamp: string | number) { 23 | // convert to ISO string: YYYY-MM-DDTHH:mm:ss.sssZ. 24 | const converted = new Date(timestamp).toISOString(); 25 | // Remove "-"s and "."s and milliseconds 26 | return converted.split('.')[0].replace(/-|:/g, '').concat('Z'); 27 | } 28 | 29 | /** 30 | * Check whether timestamp predates request or is too old 31 | * @param timestamp of ISO8601Timestamp format without 32 | * dashes or colons, e.g. 20160202T220410Z 33 | * @param expiry - number of seconds signature should be valid 34 | * @param log - log for request 35 | * @return true if there is a time problem 36 | */ 37 | export function checkTimeSkew(timestamp: string, expiry: number, log: RequestLogger) { 38 | const currentTime = Date.now(); 39 | const fifteenMinutes = (15 * 60 * 1000); 40 | const parsedTimestamp = convertAmzTimeToMs(timestamp); 41 | if ((currentTime + fifteenMinutes) < parsedTimestamp) { 42 | log.debug('current time pre-dates timestamp', { 43 | parsedTimestamp, 44 | currentTimeInMilliseconds: currentTime }); 45 | return true; 46 | } 47 | const expiryInMilliseconds = expiry * 1000; 48 | if (currentTime > parsedTimestamp + expiryInMilliseconds) { 49 | log.debug('signature has expired', { 50 | parsedTimestamp, 51 | expiry, 52 | currentTimeInMilliseconds: currentTime }); 53 | return true; 54 | } 55 | return false; 56 | } 57 | -------------------------------------------------------------------------------- /lib/errorUtils.ts: -------------------------------------------------------------------------------- 1 | export interface ErrorLike { 2 | message: any; 3 | code: any; 4 | stack: any; 5 | name: any; 6 | } 7 | 8 | export function reshapeExceptionError(error: ErrorLike) { 9 | const { message, code, stack, name } = error; 10 | return { message, code, stack, name }; 11 | } 12 | -------------------------------------------------------------------------------- /lib/executables/pensieveCreds/README.md: -------------------------------------------------------------------------------- 1 | # Get Pensieve Credentials Executable 2 | 3 | ## To make executable file from getPensieveCreds.js 4 | 5 | `npm install -g pkg` 6 | `pkg getPensieveCreds.js` 7 | 8 | This will build a mac, linux and windows file. 9 | If you just want linux, for example: 10 | `pkg getPensieveCreds.js --targets node6-linux-x64` 11 | 12 | For further options, see https://github.com/zeit/pkg 13 | 14 | ## To run the executable file 15 | 16 | Call the output executable file with an 17 | argument that names the service you 18 | are trying to get credentials for (e.g., clueso): 19 | 20 | `./getPensieveCreds-linux serviceName` 21 | -------------------------------------------------------------------------------- /lib/executables/pensieveCreds/getPensieveCreds.js: -------------------------------------------------------------------------------- 1 | const async = require('async'); 2 | const MetadataFileClient = 3 | require('../../storage/metadata/file/MetadataFileClient'); 4 | const mdClient = new MetadataFileClient({ 5 | host: 's3-metadata', 6 | port: '9993', 7 | }); 8 | const { loadOverlayVersion, parseServiceCredentials } = require('./utils'); 9 | 10 | const serviceName = process.argv[2]; 11 | if (serviceName === undefined) { 12 | throw new Error('Missing service name (e.g., clueso)'); 13 | } 14 | const tokenKey = 'auth/zenko/remote-management-token'; 15 | 16 | const mdDb = mdClient.openDB(error => { 17 | if (error) { 18 | throw error; 19 | } 20 | 21 | const db = mdDb.openSub('PENSIEVE'); 22 | return async.waterfall([ 23 | cb => db.get('configuration/overlay-version', {}, cb), 24 | (version, cb) => loadOverlayVersion(db, version, cb), 25 | (conf, cb) => db.get(tokenKey, {}, (err, instanceAuth) => { 26 | if (err) { 27 | return cb(err); 28 | } 29 | const creds = parseServiceCredentials(conf, instanceAuth, 30 | serviceName); 31 | return cb(null, creds); 32 | }), 33 | ], (err, creds) => { 34 | db.disconnect(); 35 | if (err) { 36 | throw err; 37 | } 38 | if (!creds) { 39 | throw new Error('No credentials found'); 40 | } 41 | process.stdout.write(`export AWS_ACCESS_KEY_ID="${creds.accessKey}"\n`); 42 | process.stdout 43 | .write(`export AWS_SECRET_ACCESS_KEY="${creds.secretKey}"`); 44 | }); 45 | }); 46 | -------------------------------------------------------------------------------- /lib/executables/pensieveCreds/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pensievecreds", 3 | "version": "1.0.0", 4 | "description": "Executable tool for Pensieve", 5 | "main": "getPensieveCreds.js", 6 | "scripts": { 7 | "test": "mocha --recursive --timeout 5500 tests/unit" 8 | }, 9 | "dependencies": { 10 | "mocha": "5.2.0", 11 | "async": "~2.6.1", 12 | "node-forge": "^0.7.1" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /lib/executables/pensieveCreds/tests/resources.json: -------------------------------------------------------------------------------- 1 | { 2 | "privateKey": "-----BEGIN RSA PRIVATE KEY-----\r\nMIIEowIBAAKCAQEAj13sSYE40lAX2qpBvfdGfcSVNtBf8i5FH+E8FAhORwwPu+2S\r\n3yBQbgwHq30WWxunGb1NmZL1wkVZ+vf12DtxqFRnMA08LfO4oO6oC4V8XfKeuHyJ\r\n1qlaKRINz6r9yDkTHtwWoBnlAINurlcNKgGD5p7D+G26Chbr/Oo0ZwHula9DxXy6\r\neH8/bJ5/BynyNyyWRPoAO+UkUdY5utkFCUq2dbBIhovMgjjikf5p2oWqnRKXc+JK\r\nBegr6lSHkkhyqNhTmd8+wA+8Cace4sy1ajY1t5V4wfRZea5vwl/HlyyKodvHdxng\r\nJgg6H61JMYPkplY6Gr9OryBKEAgq02zYoYTDfwIDAQABAoIBAAuDYGlavkRteCzw\r\nRU1LIVcSRWVcgIgDXTu9K8T0Ec0008Kkxomyn6LmxmroJbZ1VwsDH8s4eRH73ckA\r\nxrZxt6Pr+0lplq6eBvKtl8MtGhq1VDe+kJczjHEF6SQHOFAu/TEaPZrn2XMcGvRX\r\nO1BnRL9tepFlxm3u/06VRFYNWqqchM+tFyzLu2AuiuKd5+slSX7KZvVgdkY1ErKH\r\ngB75lPyhPb77C/6ptqUisVMSO4JhLhsD0+ekDVY982Sb7KkI+szdWSbtMx9Ek2Wo\r\ntXwJz7I8T7IbODy9aW9G+ydyhMDFmaEYIaDVFKJj5+fluNza3oQ5PtFNVE50GQJA\r\nsisGqfECgYEAwpkwt0KpSamSEH6qknNYPOwxgEuXWoFVzibko7is2tFPvY+YJowb\r\n68MqHIYhf7gHLq2dc5Jg1TTbGqLECjVxp4xLU4c95KBy1J9CPAcuH4xQLDXmeLzP\r\nJ2YgznRocbzAMCDAwafCr3uY9FM7oGDHAi5bE5W11xWx+9MlFExL3JkCgYEAvJp5\r\nf+JGN1W037bQe2QLYUWGszewZsvplnNOeytGQa57w4YdF42lPhMz6Kc/zdzKZpN9\r\njrshiIDhAD5NCno6dwqafBAW9WZl0sn7EnlLhD4Lwm8E9bRHnC9H82yFuqmNrzww\r\nzxBCQogJISwHiVz4EkU48B283ecBn0wT/fAa19cCgYEApKWsnEHgrhy1IxOpCoRh\r\nUhqdv2k1xDPN/8DUjtnAFtwmVcLa/zJopU/Zn4y1ZzSzjwECSTi+iWZRQ/YXXHPf\r\nl92SFjhFW92Niuy8w8FnevXjF6T7PYiy1SkJ9OR1QlZrXc04iiGBDazLu115A7ce\r\nanACS03OLw+CKgl6Q/RR83ECgYBCUngDVoimkMcIHHt3yJiP3ikeAKlRnMdJlsa0\r\nXWVZV4hCG3lDfRXsnEgWuimftNKf+6GdfYSvQdLdiQsCcjT5A4uLsQTByv5nf4uA\r\n1ZKOsFrmRrARzxGXhLDikvj7yP//7USkq+0BBGFhfuAvl7fMhPceyPZPehqB7/jf\r\nxX1LBQKBgAn5GgSXzzS0e06ZlP/VrKxreOHa5Z8wOmqqYQ0QTeczAbNNmuITdwwB\r\nNkbRqpVXRIfuj0BQBegAiix8om1W4it0cwz54IXBwQULxJR1StWxj3jo4QtpMQ+z\r\npVPdB1Ilb9zPV1YvDwRfdS1xsobzznAx56ecsXduZjs9mF61db8Q\r\n-----END RSA PRIVATE KEY-----\r\n", 3 | "publicKey": "-----BEGIN PUBLIC KEY-----\r\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAj13sSYE40lAX2qpBvfdG\r\nfcSVNtBf8i5FH+E8FAhORwwPu+2S3yBQbgwHq30WWxunGb1NmZL1wkVZ+vf12Dtx\r\nqFRnMA08LfO4oO6oC4V8XfKeuHyJ1qlaKRINz6r9yDkTHtwWoBnlAINurlcNKgGD\r\n5p7D+G26Chbr/Oo0ZwHula9DxXy6eH8/bJ5/BynyNyyWRPoAO+UkUdY5utkFCUq2\r\ndbBIhovMgjjikf5p2oWqnRKXc+JKBegr6lSHkkhyqNhTmd8+wA+8Cace4sy1ajY1\r\nt5V4wfRZea5vwl/HlyyKodvHdxngJgg6H61JMYPkplY6Gr9OryBKEAgq02zYoYTD\r\nfwIDAQAB\r\n-----END PUBLIC KEY-----\r\n", 4 | "accessKey": "QXP3VDG3SALNBX2QBJ1C", 5 | "secretKey": "K5FyqZo5uFKfw9QBtn95o6vuPuD0zH/1seIrqPKqGnz8AxALNSx6EeRq7G1I6JJpS1XN13EhnwGn2ipsml3Uf2fQ00YgEmImG8wzGVZm8fWotpVO4ilN4JGyQCah81rNX4wZ9xHqDD7qYR5MyIERxR/osoXfctOwY7GGUjRKJfLOguNUlpaovejg6mZfTvYAiDF+PTO1sKUYqHt1IfKQtsK3dov1EFMBB5pWM7sVfncq/CthKN5M+VHx9Y87qdoP3+7AW+RCBbSDOfQgxvqtS7PIAf10mDl8k2kEURLz+RqChu4O4S0UzbEmtja7wa7WYhYKv/tM/QeW7kyNJMmnPg==", 6 | "decryptedSecretKey": "n7PSZ3U6SgerF9PCNhXYsq3S3fRKVGdZTicGV8Ur" 7 | } -------------------------------------------------------------------------------- /lib/executables/pensieveCreds/tests/unit/utilsSpec.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const { parseServiceCredentials, decryptSecret } = 3 | require('../../utils'); 4 | const { privateKey, accessKey, secretKey, decryptedSecretKey } 5 | = require('../resources.json'); 6 | 7 | describe('decyrptSecret', () => { 8 | it('should decrypt a secret', () => { 9 | const instanceCredentials = { 10 | privateKey, 11 | }; 12 | const result = decryptSecret(instanceCredentials, secretKey); 13 | assert.strictEqual(result, decryptedSecretKey); 14 | }); 15 | }); 16 | 17 | describe('parseServiceCredentials', () => { 18 | const conf = { 19 | users: [{ accessKey, 20 | accountType: 'service-clueso', 21 | secretKey, 22 | userName: 'Search Service Account' }], 23 | }; 24 | const auth = JSON.stringify({ privateKey }); 25 | 26 | it('should parse service credentials', () => { 27 | const result = parseServiceCredentials(conf, auth, 'clueso'); 28 | const expectedResult = { 29 | accessKey, 30 | secretKey: decryptedSecretKey, 31 | }; 32 | assert.deepStrictEqual(result, expectedResult); 33 | }); 34 | 35 | it('should return undefined if no such service', () => { 36 | const result = parseServiceCredentials(conf, auth, undefined); 37 | assert.strictEqual(result, undefined); 38 | }); 39 | }); 40 | -------------------------------------------------------------------------------- /lib/executables/pensieveCreds/utils.js: -------------------------------------------------------------------------------- 1 | const forge = require('node-forge'); 2 | 3 | function decryptSecret(instanceCredentials, secret) { 4 | const privateKey = forge.pki.privateKeyFromPem( 5 | instanceCredentials.privateKey); 6 | const encryptedSecretKey = forge.util.decode64(secret); 7 | return privateKey.decrypt(encryptedSecretKey, 'RSA-OAEP', { 8 | md: forge.md.sha256.create(), 9 | }); 10 | } 11 | 12 | function loadOverlayVersion(db, version, cb) { 13 | db.get(`configuration/overlay/${version}`, {}, (err, val) => { 14 | if (err) { 15 | return cb(err); 16 | } 17 | return cb(null, JSON.parse(val)); 18 | }); 19 | } 20 | 21 | function parseServiceCredentials(conf, auth, serviceName) { 22 | const instanceAuth = JSON.parse(auth); 23 | const serviceAccount = (conf.users || []).find( 24 | u => u.accountType === `service-${serviceName}`); 25 | if (!serviceAccount) { 26 | return undefined; 27 | } 28 | return { 29 | accessKey: serviceAccount.accessKey, 30 | secretKey: decryptSecret(instanceAuth, serviceAccount.secretKey), 31 | }; 32 | } 33 | 34 | module.exports = { 35 | decryptSecret, 36 | loadOverlayVersion, 37 | parseServiceCredentials, 38 | }; 39 | -------------------------------------------------------------------------------- /lib/https/ciphers.ts: -------------------------------------------------------------------------------- 1 | export const ciphers = [ 2 | 'DHE-RSA-AES128-GCM-SHA256', 3 | 'ECDHE-ECDSA-AES128-GCM-SHA256', 4 | 'ECDHE-RSA-AES256-GCM-SHA384', 5 | 'ECDHE-ECDSA-AES256-GCM-SHA384', 6 | 'DHE-RSA-AES128-GCM-SHA256', 7 | 'ECDHE-RSA-AES128-SHA256', 8 | 'DHE-RSA-AES128-SHA256', 9 | 'ECDHE-RSA-AES256-SHA384', 10 | 'DHE-RSA-AES256-SHA384', 11 | 'ECDHE-RSA-AES256-SHA256', 12 | 'DHE-RSA-AES256-SHA256', 13 | 'HIGH', 14 | '!aNULL', 15 | '!eNULL', 16 | '!EXPORT', 17 | '!DES', 18 | '!RC4', 19 | '!MD5', 20 | '!SHA1', 21 | '!PSK', 22 | '!aECDH', 23 | '!SRP', 24 | '!IDEA', 25 | '!EDH-DSS-DES-CBC3-SHA', 26 | '!EDH-RSA-DES-CBC3-SHA', 27 | '!KRB5-DES-CBC3-SHA', 28 | ].join(':'); 29 | -------------------------------------------------------------------------------- /lib/https/dh2048.ts: -------------------------------------------------------------------------------- 1 | /* 2 | PKCS#3 DH Parameters: (2048 bit) 3 | prime: 4 | 00:87:df:53:ef:b2:86:36:e8:98:f4:de:b1:ac:22: 5 | 77:40:db:f8:48:50:03:4d:ad:c2:0f:ed:55:31:30: 6 | 1d:44:92:c7:50:da:60:94:1f:a2:02:84:d8:88:b0: 7 | c5:66:0b:53:0a:9c:74:65:95:03:f8:93:37:aa:20: 8 | 99:cb:43:8a:e7:f6:46:95:50:fb:b1:99:b1:8d:1b: 9 | 5d:a5:52:b8:a8:83:ed:c1:ab:fc:b7:42:7b:73:60: 10 | 8d:7d:41:2a:c9:16:c9:17:8a:44:f5:97:1d:41:17: 11 | 93:e6:9f:e5:96:6c:a1:41:db:ea:e9:c1:c7:f9:c2: 12 | 89:93:ad:c2:e8:31:d1:56:84:ad:b2:7b:14:72:f2: 13 | 9e:db:73:cb:19:9b:a5:2a:0f:07:dd:e4:41:c4:76: 14 | a6:1e:49:b2:b8:45:43:b6:83:61:30:8a:09:38:db: 15 | 1d:5d:2a:68:e4:68:1c:0f:81:10:30:cf:31:6f:fa: 16 | ac:9c:2c:67:e9:02:06:4c:1b:dc:1e:c9:31:b6:54: 17 | d9:39:f5:0f:93:85:d0:e9:86:f7:b5:08:b6:4e:ea: 18 | f3:91:01:cb:96:7e:14:ee:9f:c6:66:cf:83:fb:a0: 19 | f7:4a:04:8f:aa:be:8f:6c:bc:4a:b3:28:0a:ef:bb: 20 | 6d:8e:be:b5:73:12:e8:0c:97:86:77:92:f9:87:50: 21 | 8f:9b 22 | generator: 2 (0x2) 23 | -----BEGIN DH PARAMETERS----- 24 | MIIBCAKCAQEAh99T77KGNuiY9N6xrCJ3QNv4SFADTa3CD+1VMTAdRJLHUNpglB+i 25 | AoTYiLDFZgtTCpx0ZZUD+JM3qiCZy0OK5/ZGlVD7sZmxjRtdpVK4qIPtwav8t0J7 26 | c2CNfUEqyRbJF4pE9ZcdQReT5p/llmyhQdvq6cHH+cKJk63C6DHRVoStsnsUcvKe 27 | 23PLGZulKg8H3eRBxHamHkmyuEVDtoNhMIoJONsdXSpo5GgcD4EQMM8xb/qsnCxn 28 | 6QIGTBvcHskxtlTZOfUPk4XQ6Yb3tQi2TurzkQHLln4U7p/GZs+D+6D3SgSPqr6P 29 | bLxKsygK77ttjr61cxLoDJeGd5L5h1CPmwIBAg== 30 | -----END DH PARAMETERS----- 31 | */ 32 | 33 | export const dhparam = 34 | 'MIIBCAKCAQEAh99T77KGNuiY9N6xrCJ3QNv4SFADTa3CD+1VMTAdRJLHUNpglB+i' + 35 | 'AoTYiLDFZgtTCpx0ZZUD+JM3qiCZy0OK5/ZGlVD7sZmxjRtdpVK4qIPtwav8t0J7' + 36 | 'c2CNfUEqyRbJF4pE9ZcdQReT5p/llmyhQdvq6cHH+cKJk63C6DHRVoStsnsUcvKe' + 37 | '23PLGZulKg8H3eRBxHamHkmyuEVDtoNhMIoJONsdXSpo5GgcD4EQMM8xb/qsnCxn' + 38 | '6QIGTBvcHskxtlTZOfUPk4XQ6Yb3tQi2TurzkQHLln4U7p/GZs+D+6D3SgSPqr6P' + 39 | 'bLxKsygK77ttjr61cxLoDJeGd5L5h1CPmwIBAg=='; 40 | -------------------------------------------------------------------------------- /lib/https/index.ts: -------------------------------------------------------------------------------- 1 | export * as ciphers from './ciphers'; 2 | export * as dhparam from './dh2048'; 3 | -------------------------------------------------------------------------------- /lib/ipCheck.ts: -------------------------------------------------------------------------------- 1 | import ipaddr from 'ipaddr.js'; 2 | 3 | /** 4 | * checkIPinRangeOrMatch checks whether a given ip address is in an ip address 5 | * range or matches the given ip address 6 | * @param cidr - ip address range or ip address 7 | * @param ip - parsed ip address 8 | * @return true if in range, false if not 9 | */ 10 | export function checkIPinRangeOrMatch( 11 | cidr: string, 12 | ip: ipaddr.IPv4 | ipaddr.IPv6, 13 | ): boolean { 14 | // If there is an exact match of the ip address, no need to check ranges 15 | if (ip.toString() === cidr) { 16 | return true; 17 | } 18 | try { 19 | if (ip instanceof ipaddr.IPv6) { 20 | const range = ipaddr.IPv6.parseCIDR(cidr); 21 | return ip.match(range); 22 | } else { 23 | const range = ipaddr.IPv4.parseCIDR(cidr); 24 | return ip.match(range); 25 | } 26 | } catch { 27 | return false; 28 | } 29 | } 30 | 31 | /** 32 | * Parse IP address into object representation 33 | * @param ip - IPV4/IPV6/IPV4-mapped IPV6 address 34 | * @return parsedIp - Object representation of parsed IP 35 | */ 36 | export function parseIp(ip: string): ipaddr.IPv4 | ipaddr.IPv6 | {} { 37 | try { 38 | return ipaddr.IPv4.parse(ip); 39 | } catch { 40 | try { 41 | const addr = ipaddr.IPv6.parse(ip); 42 | // Copy ipaddr.process to bypass a duplicate Ipv6 parsing for isValid 43 | if (addr.kind() === 'ipv6' && addr.isIPv4MappedAddress()) { 44 | return addr.toIPv4Address(); 45 | } else { 46 | return addr; 47 | } 48 | } catch { 49 | return {}; 50 | } 51 | } 52 | } 53 | 54 | /** 55 | * Checks if an IP adress matches a given list of CIDR ranges 56 | * @param cidrList - List of CIDR ranges 57 | * @param ip - IP address 58 | * @return - true if there is match or false for no match 59 | */ 60 | export function ipMatchCidrList(cidrList: string[], ip: string): boolean { 61 | const parsedIp = parseIp(ip); 62 | return cidrList.some(item => { 63 | let cidr: string | undefined; 64 | // patch the cidr if range is not specified 65 | if (item.indexOf('/') === -1) { 66 | if (item.startsWith('127.')) { 67 | cidr = `${item}/8`; 68 | } else if (ipaddr.IPv4.isValid(item)) { 69 | cidr = `${item}/32`; 70 | } 71 | } 72 | return ( 73 | (parsedIp instanceof ipaddr.IPv4 || 74 | parsedIp instanceof ipaddr.IPv6) && 75 | checkIPinRangeOrMatch(cidr || item, parsedIp) 76 | ); 77 | }); 78 | } 79 | -------------------------------------------------------------------------------- /lib/jsutil.ts: -------------------------------------------------------------------------------- 1 | import * as util from 'util'; 2 | const debug = util.debuglog('jsutil'); 3 | 4 | // JavaScript utility functions 5 | 6 | /** 7 | * force func to be called only once, even if actually called 8 | * multiple times. The cached result of the first call is then 9 | * returned (if any). 10 | * 11 | * @note underscore.js provides this functionality but not worth 12 | * adding a new dependency for such a small use case. 13 | * 14 | * @param func function to call at most once 15 | 16 | * @return a callable wrapper mirroring func but 17 | * only calls func at first invocation. 18 | */ 19 | export function once(func: (...args: any[]) => T): (...args: any[]) => T { 20 | type State = { called: boolean; res: any }; 21 | const state: State = { called: false, res: undefined }; 22 | return function wrapper(...args: any[]) { 23 | if (!state.called) { 24 | state.called = true; 25 | state.res = func.apply(func, args); 26 | } else { 27 | const m1 = 'function already called:'; 28 | const m2 = 'returning cached result:'; 29 | debug(m1, func, m2, state.res); 30 | } 31 | return state.res; 32 | }; 33 | } 34 | -------------------------------------------------------------------------------- /lib/metrics/ZenkoMetrics.ts: -------------------------------------------------------------------------------- 1 | import promClient from 'prom-client'; 2 | 3 | export default class ZenkoMetrics { 4 | static createCounter(params: promClient.CounterConfiguration) { 5 | return new promClient.Counter(params); 6 | } 7 | 8 | static createGauge(params: promClient.GaugeConfiguration) { 9 | return new promClient.Gauge(params); 10 | } 11 | 12 | static createHistogram(params: promClient.HistogramConfiguration) { 13 | return new promClient.Histogram(params); 14 | } 15 | 16 | static createSummary(params: promClient.SummaryConfiguration) { 17 | return new promClient.Summary(params); 18 | } 19 | 20 | static getMetric(name: string) { 21 | return promClient.register.getSingleMetric(name); 22 | } 23 | 24 | static async asPrometheus() { 25 | return promClient.register.metrics(); 26 | } 27 | 28 | static asPrometheusContentType() { 29 | return promClient.register.contentType; 30 | } 31 | 32 | static collectDefaultMetrics() { 33 | return promClient.collectDefaultMetrics(); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /lib/metrics/index.ts: -------------------------------------------------------------------------------- 1 | export { default as StatsClient } from './StatsClient'; 2 | export { default as StatsModel } from './StatsModel'; 3 | export { default as RedisClient } from './RedisClient'; 4 | export { default as ZenkoMetrics } from './ZenkoMetrics'; 5 | -------------------------------------------------------------------------------- /lib/models/ObjectMDAmzRestore.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Code based on Yutaka Oishi (Fujifilm) contributions 3 | * Date: 11 Sep 2020 4 | */ 5 | 6 | /** 7 | * class representing the x-amz-restore of object metadata. 8 | * 9 | * @class 10 | */ 11 | export default class ObjectMDAmzRestore { 12 | 'expiry-date': Date | string; 13 | 'ongoing-request': boolean; 14 | 'content-md5': string; 15 | 16 | /** 17 | * 18 | * @constructor 19 | * @param ongoingRequest ongoing-request 20 | * @param [expiryDate] expiry-date 21 | * @throws case of invalid parameter 22 | */ 23 | constructor(ongoingRequest: boolean, expiryDate?: Date | string) { 24 | this.setOngoingRequest(ongoingRequest); 25 | this.setExpiryDate(expiryDate); 26 | } 27 | 28 | /** 29 | * 30 | * @param data archiveInfo 31 | * @returns true if the provided object is valid 32 | */ 33 | static isValid(data: { 'ongoing-request': boolean; 'expiry-date': Date | string }) { 34 | try { 35 | new ObjectMDAmzRestore(data['ongoing-request'], data['expiry-date']); 36 | return true; 37 | } catch { 38 | return false; 39 | } 40 | } 41 | 42 | /** 43 | * 44 | * @returns ongoing-request 45 | */ 46 | getOngoingRequest() { 47 | return this['ongoing-request']; 48 | } 49 | 50 | /** 51 | * 52 | * @param value ongoing-request 53 | * @throws case of invalid parameter 54 | */ 55 | setOngoingRequest(value?: boolean) { 56 | if (value === undefined) { 57 | throw new Error('ongoing-request is required.'); 58 | } else if (typeof value !== 'boolean') { 59 | throw new Error('ongoing-request must be type of boolean.'); 60 | } 61 | this['ongoing-request'] = value; 62 | } 63 | 64 | /** 65 | * 66 | * @returns expiry-date 67 | */ 68 | getExpiryDate() { 69 | return this['expiry-date']; 70 | } 71 | 72 | /** 73 | * 74 | * @param value expiry-date 75 | * @throws case of invalid parameter 76 | */ 77 | setExpiryDate(value?: Date | string) { 78 | if (value) { 79 | const checkWith = (new Date(value)).getTime(); 80 | if (Number.isNaN(Number(checkWith))) { 81 | throw new Error('expiry-date is must be a valid Date.'); 82 | } 83 | this['expiry-date'] = value; 84 | } 85 | } 86 | 87 | /** 88 | * 89 | * @returns itself 90 | */ 91 | getValue() { 92 | return this; 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /lib/models/index.ts: -------------------------------------------------------------------------------- 1 | export { default as ARN } from './ARN'; 2 | export { default as BackendInfo } from './BackendInfo'; 3 | export { default as BucketAzureInfo } from './BucketAzureInfo'; 4 | export { default as BucketInfo } from './BucketInfo'; 5 | export { default as BucketPolicy } from './BucketPolicy'; 6 | export { default as LifecycleConfiguration, ValidLifecycleRules } from './LifecycleConfiguration'; 7 | export { default as LifecycleRule } from './LifecycleRule'; 8 | export { default as NotificationConfiguration } from './NotificationConfiguration'; 9 | export { default as ObjectLockConfiguration } from './ObjectLockConfiguration'; 10 | export { default as ObjectMD } from './ObjectMD'; 11 | export { default as ObjectMDAmzRestore } from './ObjectMDAmzRestore'; 12 | export { default as ObjectMDArchive } from './ObjectMDArchive'; 13 | export { default as ObjectMDAzureInfo } from './ObjectMDAzureInfo'; 14 | export { default as ObjectMDLocation } from './ObjectMDLocation'; 15 | export { default as ReplicationConfiguration } from './ReplicationConfiguration'; 16 | export * as WebsiteConfiguration from './WebsiteConfiguration'; 17 | -------------------------------------------------------------------------------- /lib/network/index.ts: -------------------------------------------------------------------------------- 1 | import server from './http/server'; 2 | import * as utils from './http/utils'; 3 | import RESTServer from './rest/RESTServer'; 4 | import RESTClient from './rest/RESTClient'; 5 | import * as ProbeServer from './probe/ProbeServer'; 6 | import HealthProbeServer from './probe/HealthProbeServer'; 7 | import * as Utils from './probe/Utils'; 8 | 9 | export const http = { server, utils }; 10 | export const rest = { RESTServer, RESTClient }; 11 | export const probe = { ProbeServer, HealthProbeServer, Utils }; 12 | 13 | export { default as RoundRobin } from './RoundRobin'; 14 | export { default as kmip } from './kmip'; 15 | export { default as kmipClient } from './kmip/Client'; 16 | export { default as kmipClusterClient } from './kmip/ClusterClient'; 17 | export { default as KmsAWSClient } from './kmsAWS/Client'; 18 | export * as rpc from './rpc/rpc'; 19 | export * as level from './rpc/level-net'; 20 | -------------------------------------------------------------------------------- /lib/network/kmip/Message.ts: -------------------------------------------------------------------------------- 1 | import assert from 'assert'; 2 | 3 | function _lookup(decodedTTLV: any[], path: string) { 4 | const xpath = path.split('/').filter(word => word.length > 0); 5 | const canonicalPath = xpath.join('/'); 6 | const obj = decodedTTLV; 7 | let res: any[] = []; 8 | assert(Array.isArray(obj)); 9 | for (let current = xpath.shift(); current; current = xpath.shift()) { 10 | for (let i = 0; i < obj.length; ++i) { 11 | const cell = obj[i]; 12 | if (cell[current]) { 13 | if (xpath.length === 0) { 14 | /* Skip if the search path has not been 15 | * completely consumed yet */ 16 | res.push(cell[current].value); 17 | } else { 18 | const subPath = xpath.join('/'); 19 | assert(current.length + 1 + subPath.length === 20 | canonicalPath.length); 21 | const intermediate = 22 | _lookup(cell[current].value, subPath); 23 | res = res.concat(intermediate); 24 | } 25 | } 26 | } 27 | } 28 | return res; 29 | } 30 | 31 | export default class Message { 32 | content: any[]; 33 | 34 | /** 35 | * Construct a new abstract Message 36 | * @param content - the content of the message 37 | */ 38 | constructor(content: any[]) { 39 | this.content = content; 40 | } 41 | 42 | /** 43 | * Lookup the values corresponding to the provided path 44 | * @param path - the path in the hierarchy of the values 45 | * of interest 46 | * @return - an array of the values matching the provided path 47 | */ 48 | lookup(path: string) { 49 | return _lookup(this.content, path); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /lib/network/kmip/errorMapping.ts: -------------------------------------------------------------------------------- 1 | import { errorInstances } from '../../errors'; 2 | 3 | /** 4 | * Format to map kmip error response to arsenal kms error 5 | * 6 | * ```json 7 | * { 8 | * [resultStatus]: { 9 | * [resultReason]: { 10 | * [/resultMessage/]: arsenal_kms_error 11 | * } 12 | * } 13 | * } 14 | * ``` 15 | * @see https://docs.oasis-open.org/kmip/spec/v1.4/errata01/os/kmip-spec-v1.4-errata01-os-redlined.html#_Toc490660974 16 | * 17 | * KMS has specific error codes, but KMIP resultReason is not enough 18 | * to map to KMS errors and resultMessage format is not enforced. 19 | * So for most errors we'll use Access Denied and include 20 | * the KMIP resultReason and resultMessage in description 21 | */ 22 | export const errorMapping = { 23 | 'Operation Failed': { 24 | 'Item Not Found': errorInstances['KMS.NotFoundException'], 25 | } 26 | }; 27 | 28 | /** 29 | * Produce a generic error message to return to client for kmip error 30 | * @param operation kmip operation 31 | * @param resource keyId or bucketName 32 | * @returns msg to use in error.customizeDescription 33 | */ 34 | export function kmipMsg(operation: string, resource: string | null | undefined, detail: string) { 35 | return `KMS (KMIP) error for ${operation}${resource ? ` on ${resource}` : ''}. ${detail}`; 36 | } 37 | -------------------------------------------------------------------------------- /lib/network/kmip/transport/tls.ts: -------------------------------------------------------------------------------- 1 | import tls from 'tls'; 2 | import TransportTemplate, { Options } from './TransportTemplate'; 3 | 4 | export default class TlsTransport extends TransportTemplate { 5 | constructor(options: Options) { 6 | super(tls, options); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /lib/network/probe/Utils.ts: -------------------------------------------------------------------------------- 1 | import * as http from 'http'; 2 | 3 | import type { RequestLogger } from 'werelogs'; 4 | 5 | import { ArsenalError } from '../../errors'; 6 | 7 | /** 8 | * Send a successful HTTP response of 200 OK 9 | * @param res - HTTP response for writing 10 | * @param log - Werelogs instance for logging if you choose to 11 | * @param [message] - Message to send as response, defaults to OK 12 | */ 13 | export function sendSuccess( 14 | res: http.ServerResponse, 15 | log: RequestLogger, 16 | message = 'OK' 17 | ) { 18 | log.debug('replying with success'); 19 | res.writeHead(200); 20 | res.end(message); 21 | } 22 | 23 | /** 24 | * Send an Arsenal Error response 25 | * @param res - HTTP response for writing 26 | * @param log - Werelogs instance for logging if you choose to 27 | * @param error - Error to send back to the user 28 | * @param [optMessage] - Message to use instead of the errors message 29 | */ 30 | export function sendError( 31 | res: http.ServerResponse, 32 | log: RequestLogger, 33 | error: ArsenalError, 34 | optMessage?: string 35 | ) { 36 | const message = optMessage || error.description || ''; 37 | log.debug('sending back error response', { 38 | httpCode: error.code, 39 | errorType: error.message, 40 | error: message, 41 | }); 42 | res.writeHead(error.code); 43 | res.end( 44 | JSON.stringify({ 45 | errorType: error.message, 46 | errorMessage: message, 47 | }) 48 | ); 49 | } 50 | -------------------------------------------------------------------------------- /lib/network/rest/utils.ts: -------------------------------------------------------------------------------- 1 | import { errorInstances } from '../../errors'; 2 | import * as constants from '../../constants'; 3 | import * as url from 'url'; 4 | const passthroughPrefixLength = constants.passthroughFileURL.length; 5 | 6 | export function explodePath(path: string) { 7 | if (path.startsWith(constants.passthroughFileURL)) { 8 | const key = path.slice(passthroughPrefixLength + 1); 9 | return { 10 | service: constants.passthroughFileURL, 11 | key: key.length > 0 ? key : undefined, 12 | }; 13 | } 14 | const pathMatch = /^(\/[a-zA-Z0-9]+)(\/([0-9a-f]*))?$/.exec(path); 15 | if (pathMatch) { 16 | return { 17 | service: pathMatch[1], 18 | key: (pathMatch[3] !== undefined && pathMatch[3].length > 0 ? 19 | pathMatch[3] : undefined), 20 | }; 21 | } 22 | throw errorInstances.InvalidURI.customizeDescription('malformed URI'); 23 | } 24 | 25 | /** 26 | * Parse the given url and return a pathInfo object. Sanity checks are 27 | * performed. 28 | * 29 | * @param urlStr - URL to parse 30 | * @param expectKey - whether the command expects to see a 31 | * key in the URL 32 | * @return a pathInfo object with URL items containing the 33 | * following attributes: 34 | * - pathInfo.service {String} - The name of REST service ("DataFile") 35 | * - pathInfo.key {String} - The requested key 36 | */ 37 | export function parseURL(urlStr: string, expectKey: boolean) { 38 | const urlObj = url.parse(urlStr); 39 | const pathInfo = explodePath(decodeURI(urlObj.path!)); 40 | if ((pathInfo.service !== constants.dataFileURL) 41 | && (pathInfo.service !== constants.passthroughFileURL)) { 42 | throw errorInstances.InvalidAction.customizeDescription( 43 | `unsupported service '${pathInfo.service}'`); 44 | } 45 | if (expectKey && pathInfo.key === undefined) { 46 | throw errorInstances.MissingParameter.customizeDescription( 47 | 'URL is missing key'); 48 | } 49 | if (!expectKey && pathInfo.key !== undefined) { 50 | // note: we may implement rewrite functionality by allowing a 51 | // key in the URL, though we may still provide the new key in 52 | // the Location header to keep immutability property and 53 | // atomicity of the update (we would just remove the old 54 | // object when the new one has been written entirely in this 55 | // case, saving a request over an equivalent PUT + DELETE). 56 | throw errorInstances.InvalidURI.customizeDescription( 57 | 'PUT url cannot contain a key'); 58 | } 59 | return pathInfo; 60 | } 61 | -------------------------------------------------------------------------------- /lib/network/rpc/utils.ts: -------------------------------------------------------------------------------- 1 | import { ArsenalError, allowUnsafeErrComp } from '../../errors'; 2 | 3 | // eslint-disable-line 4 | 5 | /** 6 | * @brief turn all err own and prototype attributes into own attributes 7 | * 8 | * This is done so that JSON.stringify() can properly serialize those 9 | * attributes (e.g. err.notFound) 10 | * 11 | * @param err error object 12 | * @return flattened object containing err attributes 13 | */ 14 | export function flattenError(err: Error) { 15 | if (!err) { 16 | return err; 17 | } 18 | 19 | if (err instanceof ArsenalError) { 20 | return err.flatten(); 21 | } 22 | 23 | const flattenedErr = {}; 24 | 25 | // TODO fix this 26 | // @ts-expect-errors 27 | flattenedErr.message = err.message; 28 | for (const k in err) { 29 | if (!(k in flattenedErr)) { 30 | flattenedErr[k] = err[k]; 31 | } 32 | } 33 | return flattenedErr; 34 | }; 35 | 36 | /** 37 | * @brief recreate a proper Error object from its flattened 38 | * representation created with flattenError(). 39 | * 40 | * @note Its internals may differ from the original Error object but 41 | * its attributes should be the same. 42 | * 43 | * @param err flattened error object 44 | * @return a reconstructed Error object inheriting err 45 | * attributes 46 | */ 47 | export function reconstructError(err: Error) { 48 | if (!err) { 49 | return err; 50 | } 51 | 52 | const arsenalFlat = ArsenalError.unflatten(err); 53 | if (arsenalFlat !== null) { 54 | return arsenalFlat; 55 | } 56 | 57 | const reconstructedErr = new Error(err.message); 58 | // This restores the old behavior of errors. This should be removed as soon 59 | // as all dependent codebases have been migrated to `is` accessors (ARSN-176). 60 | reconstructedErr[err.message] = true; 61 | if (allowUnsafeErrComp){ 62 | // @ts-expect-error 63 | reconstructedErr.is = { [err.message]: true }; 64 | } 65 | Object.keys(err).forEach(k => { 66 | reconstructedErr[k] = err[k]; 67 | }); 68 | return reconstructedErr; 69 | }; 70 | -------------------------------------------------------------------------------- /lib/network/utils.ts: -------------------------------------------------------------------------------- 1 | import { errorInstances } from '../errors'; 2 | 3 | /** 4 | * Normalize errors according to arsenal definitions with a custom prefix 5 | * @param err - an Error instance or a message string 6 | * @param messagePrefix - prefix for the error message 7 | * @returns - arsenal error 8 | */ 9 | function _normalizeArsenalError(err: string | Error, messagePrefix: string) { 10 | if (typeof err === 'string') { 11 | return errorInstances.InternalError 12 | .customizeDescription(`${messagePrefix} ${err}`); 13 | } else if ( 14 | err instanceof Error || 15 | // INFO: The second part is here only for Jest, to remove when we'll be 16 | // fully migrated to TS 17 | // @ts-expect-error 18 | (err && typeof err.message === 'string') 19 | ) { 20 | return errorInstances.InternalError 21 | .customizeDescription(`${messagePrefix} ${err.message}`); 22 | } 23 | return errorInstances.InternalError 24 | .customizeDescription(`${messagePrefix} Unspecified error`); 25 | } 26 | 27 | export function arsenalErrorKMIP(err: string | Error) { 28 | return _normalizeArsenalError(err, 'KMIP:'); 29 | } 30 | 31 | export function arsenalErrorAWSKMS(err: string | Error) { 32 | return _normalizeArsenalError(err, 'AWS_KMS:'); 33 | } 34 | -------------------------------------------------------------------------------- /lib/policyEvaluator/requestUtils.ts: -------------------------------------------------------------------------------- 1 | import * as ipCheck from '../ipCheck'; 2 | import { IncomingMessage } from 'http'; 3 | import { TLSSocket } from 'tls'; 4 | 5 | export interface S3Config { 6 | requests: { 7 | trustedProxyCIDRs: string[], 8 | extractClientIPFromHeader: string, 9 | extractProtocolFromHeader: string, 10 | } 11 | } 12 | 13 | /** 14 | * getClientIp - Gets the client IP from the request 15 | * @param request - http request object 16 | * @param s3config - s3 config 17 | * @return - returns client IP from the request 18 | */ 19 | export function getClientIp(request: IncomingMessage, s3config?: S3Config): string { 20 | const requestConfig = s3config?.requests; 21 | const remoteAddress = request.socket.remoteAddress; 22 | const clientIp = remoteAddress?.toString() ?? ''; 23 | if (requestConfig) { 24 | const { trustedProxyCIDRs, extractClientIPFromHeader } = requestConfig; 25 | /** 26 | * if requests are configured to come via proxy, 27 | * check from config which proxies are to be trusted and 28 | * which header to be used to extract client IP 29 | */ 30 | if (ipCheck.ipMatchCidrList(trustedProxyCIDRs, clientIp)) { 31 | // Request headers in nodejs are lower-cased, so we should not 32 | // be case-sentive when looking for the header, as http headers 33 | // are case-insensitive. 34 | const ipFromHeader = request.headers[extractClientIPFromHeader.toLowerCase()]?.toString(); 35 | if (ipFromHeader && ipFromHeader.trim().length) { 36 | return ipFromHeader.split(',')[0].trim(); 37 | } 38 | } 39 | } 40 | return clientIp; 41 | } 42 | 43 | /** 44 | * getHttpProtocolSecurity - Dete²object 45 | * @param s3config - s3 config 46 | * @return {boolean} - returns true if the request is secure 47 | */ 48 | export function getHttpProtocolSecurity(request: IncomingMessage, s3config?: S3Config): boolean { 49 | const requestConfig = s3config?.requests; 50 | if (requestConfig) { 51 | const { trustedProxyCIDRs } = requestConfig; 52 | const clientIp = request.socket.remoteAddress?.toString() ?? ''; 53 | if (ipCheck.ipMatchCidrList(trustedProxyCIDRs, clientIp)) { 54 | return request.headers[requestConfig.extractProtocolFromHeader.toLowerCase()] === 'https'; 55 | } 56 | } 57 | return request.socket instanceof TLSSocket && request.socket.encrypted; 58 | } 59 | -------------------------------------------------------------------------------- /lib/policyEvaluator/utils/checkArnMatch.ts: -------------------------------------------------------------------------------- 1 | import { handleWildcardInResource } from './wildcards'; 2 | import { policyArnAllowedEmptyAccountId } from '../../constants'; 3 | /** 4 | * Checks whether an ARN from a request matches an ARN in a policy 5 | * to compare against each portion of the ARN from the request 6 | * @param policyArn - arn from policy 7 | * @param requestRelativeId - last part of the arn from the request 8 | * @param requestArnArr - all parts of request arn split on ":" 9 | * @param caseSensitive - whether the comparison should be 10 | * case sensitive 11 | * @return true if match, false if not 12 | */ 13 | export default function checkArnMatch( 14 | policyArn: string, 15 | requestRelativeId: string, 16 | requestArnArr: string[], 17 | caseSensitive: boolean, 18 | ): boolean { 19 | const regExofArn = handleWildcardInResource(policyArn); 20 | // The relativeId is the last part of the ARN (for instance, a bucket and 21 | // object name in S3) 22 | // Join on ":" in case there were ":" in the relativeID at the end 23 | // of the arn 24 | const policyRelativeId = caseSensitive ? regExofArn.slice(5).join(':') : 25 | regExofArn.slice(5).join(':').toLowerCase(); 26 | const policyRelativeIdRegEx = new RegExp(policyRelativeId); 27 | // Check to see if the relative-id matches first since most likely 28 | // to diverge. If not a match, the resource is not applicable so return 29 | // false 30 | if (!policyRelativeIdRegEx.test(caseSensitive ? 31 | requestRelativeId : requestRelativeId.toLowerCase())) { 32 | return false; 33 | } 34 | // Check the other parts of the ARN to make sure they match. If not, 35 | // return false. 36 | for (let j = 0; j < 5; j++) { 37 | const segmentRegEx = new RegExp(regExofArn[j]); 38 | const requestSegment = caseSensitive ? requestArnArr[j] : 39 | requestArnArr[j].toLowerCase(); 40 | const policyArnArr = policyArn.split(':'); 41 | // We want to allow an empty account ID for utapi and SUR service ARNs to not 42 | // break compatibility. 43 | if (j === 4 && policyArnAllowedEmptyAccountId.includes(policyArnArr[2]) 44 | && policyArnArr[4] === '') { 45 | continue; 46 | } else if (!segmentRegEx.test(requestSegment)) { 47 | return false; 48 | } 49 | } 50 | // If there were matches on all parts of the ARN, return true 51 | return true; 52 | } 53 | -------------------------------------------------------------------------------- /lib/policyEvaluator/utils/objectTags.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Removes tag key value from condition key and adds it to value if needed 3 | * @param key - condition key 4 | * @param value - condition value 5 | * @return key/value pair to use 6 | */ 7 | export function transformTagKeyValue(key: string, value: string): [string, string | string[]] { 8 | const patternKeys = ['s3:ExistingObjectTag/', 's3:RequestObjectTagKey/']; 9 | if (!patternKeys.some(k => key.includes(k))) { 10 | return [key, value]; 11 | } 12 | // if key is RequestObjectTag or ExistingObjectTag, 13 | // remove tag key from condition key and add to value 14 | // and transform value into query string 15 | const [conditionKey, tagKey] = key.split('/'); 16 | const transformedValue = [tagKey, value].join('='); 17 | return [conditionKey, [transformedValue]]; 18 | } 19 | 20 | /** 21 | * Gets array of tag key names from request tag query string 22 | * @param tagQuery - request tags in query string format 23 | * @return array of tag key names 24 | */ 25 | export function getTagKeys(tagQuery: string) { 26 | return tagQuery.split('&') 27 | .map(tag => tag.split('=')[0]); 28 | } 29 | -------------------------------------------------------------------------------- /lib/policyEvaluator/utils/wildcards.ts: -------------------------------------------------------------------------------- 1 | // * represents any combo of characters 2 | // ? represents any single character 3 | 4 | // TODO: Note that there are special rules for * in Principal. 5 | // Handle when working with bucket policies. 6 | 7 | 8 | // Replace all '*' with '.*' (allow any combo of letters) 9 | // and all '?' with '.{1}' (allow for any one character) 10 | // If *, ? or $ are enclosed in ${}, keep literal *, ?, or $ 11 | function characterMap(char: string) { 12 | const map = { 13 | '\\*': '.*?', 14 | '\\?': '.{1}', 15 | '\\$\\{\\*\\}': '\\*', 16 | '\\$\\{\\?\\}': '\\?', 17 | '\\$\\{\\$\\}': '\\$', 18 | }; 19 | return map[char]; 20 | } 21 | 22 | /** 23 | * Converts string into a string that has all regEx characters escaped except 24 | * for those needed to check for AWS wildcards. Converted string can then 25 | * be used for a regEx comparison. 26 | * @param string - any input string 27 | * @return converted string 28 | */ 29 | export const handleWildcards = (string: string) => { 30 | // Escape all regExp special characters 31 | // Then replace the AWS special characters with regExp equivalents 32 | const regExStr = string.replace(/[\\^$*+?.()|[\]{}]/g, '\\$&').replace( 33 | /(\\\*)|(\\\?)|(\\\$\\\{\\\*\\\})|(\\\$\\\{\\\?\\\})|(\\\$\\\{\\\$\\\})/g, 34 | characterMap 35 | ); 36 | return `^${regExStr}$`; 37 | }; 38 | 39 | /** 40 | * Converts each portion of an ARN into a converted regEx string 41 | * to compare against each portion of the ARN from the request 42 | * @param arn - arn for requested resource 43 | * @return array of strings to be used for regEx comparisons 44 | */ 45 | export const handleWildcardInResource = (arn: string) => { 46 | // Wildcards can be part of the resource ARN. 47 | // Wildcards do NOT span segments of the ARN (separated by ":") 48 | 49 | // Example: all elements in specific bucket: 50 | // "Resource": "arn:aws:s3:::my_corporate_bucket/*" 51 | // ARN format: 52 | // arn:partition:service:region:namespace:relative-id 53 | const arnArr = arn.split(':'); 54 | return arnArr.map(portion => handleWildcards(portion)); 55 | }; 56 | -------------------------------------------------------------------------------- /lib/s3middleware/MD5Sum.ts: -------------------------------------------------------------------------------- 1 | import { Transform } from 'stream'; 2 | import * as crypto from 'crypto'; 3 | 4 | /** 5 | * This class is design to compute md5 hash at the same time as sending 6 | * data through a stream 7 | */ 8 | export default class MD5Sum extends Transform { 9 | hash: ReturnType; 10 | completedHash?: string; 11 | 12 | constructor() { 13 | super({}); 14 | this.hash = crypto.createHash('md5'); 15 | this.completedHash = undefined; 16 | } 17 | 18 | /** 19 | * This function will update the current md5 hash with the next chunk 20 | * 21 | * @param chunk - Chunk to compute 22 | * @param encoding - Data encoding 23 | * @param callback - Callback(err, chunk, encoding) 24 | */ 25 | _transform( 26 | chunk: string, 27 | encoding: crypto.Encoding, 28 | callback: ( 29 | err: Error | null, 30 | chunk: string, 31 | encoding: crypto.Encoding, 32 | ) => void, 33 | ) { 34 | this.hash.update(chunk, encoding); 35 | callback(null, chunk, encoding); 36 | } 37 | 38 | /** 39 | * This function will end the hash computation 40 | * 41 | * @param callback(err) 42 | */ 43 | _flush(callback: (err: Error | null) => void) { 44 | this.completedHash = this.hash.digest('hex'); 45 | this.emit('hashed'); 46 | callback(null); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /lib/s3middleware/azureHelpers/ResultsCollector.ts: -------------------------------------------------------------------------------- 1 | import { EventEmitter } from 'events'; 2 | 3 | /** 4 | * Class to collect results of streaming subparts. 5 | * Emits "done" event when streaming is complete and Azure has returned 6 | * results for putting each of the subparts 7 | * Emits "error" event if Azure returns an error for putting a subpart and 8 | * streaming is in-progress 9 | * @class ResultsCollector 10 | */ 11 | export default class ResultsCollector extends EventEmitter { 12 | // TODO Add better type. 13 | _results: any[]; 14 | _queue: number; 15 | _streamingFinished: boolean; 16 | 17 | constructor() { 18 | super(); 19 | this._results = []; 20 | this._queue = 0; 21 | this._streamingFinished = false; 22 | } 23 | 24 | /** 25 | * ResultsCollector.pushResult - register result of putting one subpart 26 | * and emit "done" or "error" events if appropriate 27 | * @param err - error returned from Azure after 28 | * putting a subpart 29 | * @param subPartIndex - the index of the subpart 30 | * @emits ResultCollector#done 31 | * @emits ResultCollector#error 32 | */ 33 | pushResult(err: Error | null | undefined, subPartIndex: number) { 34 | this._results.push({ 35 | error: err, 36 | subPartIndex, 37 | }); 38 | this._queue--; 39 | if (this._resultsComplete()) { 40 | this.emit('done', err, this._results); 41 | } else if (err) { 42 | this.emit('error', err, subPartIndex); 43 | } 44 | } 45 | 46 | /** 47 | * ResultsCollector.pushOp - register operation to put another subpart 48 | */ 49 | pushOp() { 50 | this._queue++; 51 | } 52 | 53 | /** 54 | * ResultsCollector.enableComplete - register streaming has finished, 55 | * allowing ResultCollector#done event to be emitted when last result 56 | * has been returned 57 | */ 58 | enableComplete() { 59 | this._streamingFinished = true; 60 | } 61 | 62 | _resultsComplete() { 63 | return (this._queue === 0 && this._streamingFinished); 64 | } 65 | } 66 | 67 | /** 68 | * "done" event 69 | * @event ResultCollector#done 70 | * @type {(Error|undefined)} err - error returned by Azure putting last subpart 71 | * @type {object[]} results - result for putting each of the subparts 72 | * @property {Error} [results[].error] - error returned by Azure putting subpart 73 | * @property {number} results[].subPartIndex - index of the subpart 74 | */ 75 | /** 76 | * "error" event 77 | * @event ResultCollector#error 78 | * @type {(Error|undefined)} error - error returned by Azure last subpart 79 | * @type {number} subPartIndex - index of the subpart 80 | */ 81 | -------------------------------------------------------------------------------- /lib/s3middleware/escapeForXml.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Project: node-xml https://github.com/dylang/node-xml 3 | * License: MIT https://github.com/dylang/node-xml/blob/master/LICENSE 4 | */ 5 | const XML_CHARACTER_MAP = { 6 | '&': '&', 7 | '"': '"', 8 | "'": ''', 9 | '<': '<', 10 | '>': '>', 11 | }; 12 | 13 | export default function escapeForXml(string: string) { 14 | return string && string.replace 15 | ? string.replace(/([&"<>'])/g, (str, item) => XML_CHARACTER_MAP[item]) 16 | : string; 17 | } 18 | -------------------------------------------------------------------------------- /lib/s3middleware/lifecycleHelpers/index.ts: -------------------------------------------------------------------------------- 1 | export { default as LifecycleDateTime } from './LifecycleDateTime'; 2 | export { default as LifecycleUtils } from './LifecycleUtils'; 3 | -------------------------------------------------------------------------------- /lib/s3middleware/nullStream.ts: -------------------------------------------------------------------------------- 1 | import { Readable } from 'stream'; 2 | 3 | /** This class is used to produce zeros filled buffers for a reader consumption */ 4 | export default class NullStream extends Readable { 5 | bytesToRead: number; 6 | /** 7 | * Construct a new zeros filled buffers producer that will 8 | * produce as much bytes as specified by the range parameter, or the size 9 | * parameter if range is null or not constituted of 2 elements 10 | * @param size - the number of null bytes to produce 11 | * @param range - a range specification to override to size 12 | */ 13 | constructor(size: number, range?: [number, number]) { 14 | super({}); 15 | if (Array.isArray(range) && range.length === 2) { 16 | this.bytesToRead = range[1] - range[0] + 1; 17 | } else { 18 | this.bytesToRead = size; 19 | } 20 | } 21 | 22 | /** 23 | * This function generates the stream of null bytes 24 | * 25 | * @param size - advisory amount of data to produce 26 | */ 27 | _read(size: number) { 28 | const toRead = Math.min(size, this.bytesToRead); 29 | const buffer = toRead > 0 30 | ? Buffer.alloc(toRead, 0) 31 | : null; 32 | this.bytesToRead -= toRead; 33 | this.push(buffer); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /lib/s3middleware/objectUtils.ts: -------------------------------------------------------------------------------- 1 | const msInOneDay = 24 * 60 * 60 * 1000; // Milliseconds in a day. 2 | 3 | export const getMD5Buffer = (base64MD5: WithImplicitCoercion | Uint8Array) => 4 | base64MD5 instanceof Uint8Array ? base64MD5 : Buffer.from(base64MD5, 'base64'); 5 | 6 | export const getHexMD5 = (base64MD5: WithImplicitCoercion | Uint8Array) => 7 | getMD5Buffer(base64MD5).toString('hex'); 8 | 9 | export const getBase64MD5 = (hexMD5: WithImplicitCoercion) => 10 | Buffer.from(hexMD5, 'hex').toString('base64'); 11 | 12 | 13 | /** 14 | * Calculates the number of scaled milliseconds per day based on the given time progression factor. 15 | * This function is intended for testing and simulation purposes only. 16 | * @param {number} timeProgressionFactor - The desired time progression factor for scaling. 17 | * @returns {number} The number of scaled milliseconds per day. 18 | * If the result is 0, the minimum value of 1 millisecond is returned. 19 | */ 20 | export const scaleMsPerDay = (timeProgressionFactor: number): number => 21 | Math.round(msInOneDay / (timeProgressionFactor || 1)) || 1; 22 | -------------------------------------------------------------------------------- /lib/s3middleware/prepareStream.ts: -------------------------------------------------------------------------------- 1 | import V4Transform, { TransformParams } from '../auth/v4/streamingV4/V4Transform'; 2 | import Vault from '../auth/Vault'; 3 | import * as werelogs from 'werelogs'; 4 | 5 | /** 6 | * Prepares the stream if the chunks are sent in a v4 Auth request 7 | * @param stream - stream containing the data 8 | * @param streamingV4Params - if v4 auth, object containing 9 | * accessKey, signatureFromRequest, region, scopeDate, timestamp, and 10 | * credentialScope (to be used for streaming v4 auth if applicable) 11 | * @param vault - Vault instance passed from CloudServer 12 | * @param log - the current request logger 13 | * @param cb - callback containing the result for V4Transform 14 | * @return - V4Transform object if v4 Auth request, or else the stream 15 | */ 16 | export function prepareStream( 17 | stream: any, 18 | streamingV4Params: TransformParams, 19 | vault: Vault, 20 | log: werelogs.RequestLogger, 21 | cb: any, 22 | ) { 23 | if (stream.headers['x-amz-content-sha256'] === 24 | 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD') { 25 | const v4Transform = new V4Transform(streamingV4Params, vault, log, cb); 26 | stream.pipe(v4Transform); 27 | return v4Transform; 28 | } 29 | return stream; 30 | } 31 | -------------------------------------------------------------------------------- /lib/s3middleware/userMetadata.ts: -------------------------------------------------------------------------------- 1 | import * as http from 'http'; 2 | import * as constants from '../constants'; 3 | import errors from '../errors'; 4 | 5 | /** 6 | * Pull user provided meta headers from request headers 7 | * @param headers - headers attached to the http request (lowercased) 8 | * @return all user meta headers or MetadataTooLarge 9 | */ 10 | export function getMetaHeaders(headers: http.IncomingHttpHeaders) { 11 | const rawHeaders = Object.entries(headers); 12 | const filtered = rawHeaders.filter(([k]) => k.startsWith('x-amz-meta-')); 13 | const totalLength = filtered.reduce((length, [k, v]) => { 14 | if (!v) { return length; } 15 | return length + k.length + v.toString().length; 16 | }, 0); 17 | if (totalLength <= constants.maximumMetaHeadersSize) { 18 | return Object.fromEntries(filtered); 19 | } else { 20 | return errors.MetadataTooLarge; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /lib/s3routes/index.ts: -------------------------------------------------------------------------------- 1 | export { default as routes } from './routes'; 2 | export * as routesUtils from './routesUtils'; 3 | -------------------------------------------------------------------------------- /lib/s3routes/routes/routeHEAD.ts: -------------------------------------------------------------------------------- 1 | import type { RequestLogger } from 'werelogs'; 2 | 3 | import * as routesUtils from '../routesUtils'; 4 | import errors from '../../errors'; 5 | import StatsClient from '../../metrics/StatsClient'; 6 | import * as http from 'http'; 7 | 8 | export default function routeHEAD( 9 | request: http.IncomingMessage, 10 | response: http.ServerResponse, 11 | api: { callApiMethod: routesUtils.CallApiMethod }, 12 | log: RequestLogger, 13 | statsClient?: StatsClient, 14 | ) { 15 | log.debug('routing request', { method: 'routeHEAD' }); 16 | const { bucketName, objectKey } = request as any; 17 | if (bucketName === undefined) { 18 | log.trace('head request without bucketName'); 19 | routesUtils.responseXMLBody(errors.MethodNotAllowed, 20 | null, response, log); 21 | } else if (objectKey === undefined) { 22 | // HEAD bucket 23 | api.callApiMethod('bucketHead', request, response, log, 24 | (err, corsHeaders) => { 25 | routesUtils.statsReport500(err, statsClient); 26 | return routesUtils.responseNoBody(err, corsHeaders, response, 27 | 200, log); 28 | }); 29 | } else { 30 | // HEAD object 31 | api.callApiMethod('objectHead', request, response, log, 32 | (err, resHeaders) => { 33 | routesUtils.statsReport500(err, statsClient); 34 | return routesUtils.responseContentHeaders(err, {}, resHeaders, 35 | response, log); 36 | }); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /lib/s3routes/routes/routeOPTIONS.ts: -------------------------------------------------------------------------------- 1 | import type { RequestLogger } from 'werelogs'; 2 | 3 | import * as routesUtils from '../routesUtils'; 4 | import { errorInstances } from '../../errors'; 5 | import * as http from 'http'; 6 | import StatsClient from '../../metrics/StatsClient'; 7 | 8 | export default function routeOPTIONS( 9 | request: http.IncomingMessage, 10 | response: http.ServerResponse, 11 | api: { callApiMethod: routesUtils.CallApiMethod }, 12 | log: RequestLogger, 13 | statsClient?: StatsClient, 14 | ) { 15 | log.debug('routing request', { method: 'routeOPTION' }); 16 | 17 | const corsMethod = request.headers['access-control-request-method'] || null; 18 | 19 | if (!request.headers.origin) { 20 | const msg = 'Insufficient information. Origin request header needed.'; 21 | const err = errorInstances.BadRequest.customizeDescription(msg); 22 | log.debug('missing origin', { method: 'routeOPTIONS', error: err }); 23 | return routesUtils.responseXMLBody(err, null, response, log); 24 | } 25 | if (['GET', 'PUT', 'HEAD', 'POST', 'DELETE'].indexOf(corsMethod ?? '') < 0) { 26 | const msg = `Invalid Access-Control-Request-Method: ${corsMethod}`; 27 | const err = errorInstances.BadRequest.customizeDescription(msg); 28 | log.debug('invalid Access-Control-Request-Method', 29 | { method: 'routeOPTIONS', error: err }); 30 | return routesUtils.responseXMLBody(err, null, response, log); 31 | } 32 | 33 | return api.callApiMethod('corsPreflight', request, response, log, 34 | (err, resHeaders) => { 35 | routesUtils.statsReport500(err, statsClient); 36 | return routesUtils.responseNoBody(err, resHeaders, response, 200, 37 | log); 38 | }); 39 | } 40 | -------------------------------------------------------------------------------- /lib/s3routes/routes/routePOST.ts: -------------------------------------------------------------------------------- 1 | import type { RequestLogger } from 'werelogs'; 2 | 3 | import * as routesUtils from '../routesUtils'; 4 | import errors from '../../errors'; 5 | import * as http from 'http'; 6 | 7 | export default function routePOST( 8 | request: http.IncomingMessage, 9 | response: http.ServerResponse, 10 | api: { callApiMethod: routesUtils.CallApiMethod }, 11 | log: RequestLogger, 12 | ) { 13 | log.debug('routing request', { method: 'routePOST' }); 14 | 15 | const { query, bucketName, objectKey } = request as any; 16 | 17 | const invalidMultiObjectDelReq = query.delete !== undefined 18 | && bucketName === undefined; 19 | if (invalidMultiObjectDelReq) { 20 | return routesUtils.responseNoBody(errors.MethodNotAllowed, null, 21 | response, undefined, log); 22 | } 23 | 24 | // @ts-ignore 25 | request.post = ''; 26 | 27 | const invalidInitiateMpuReq = query.uploads !== undefined 28 | && objectKey === undefined; 29 | const invalidCompleteMpuReq = query.uploadId !== undefined 30 | && objectKey === undefined; 31 | if (invalidInitiateMpuReq || invalidCompleteMpuReq) { 32 | return routesUtils.responseNoBody(errors.InvalidURI, null, 33 | response, undefined, log); 34 | } 35 | 36 | // POST initiate multipart upload 37 | if (query.uploads !== undefined) { 38 | return api.callApiMethod('initiateMultipartUpload', request, 39 | response, log, (err, result, corsHeaders) => 40 | routesUtils.responseXMLBody(err, result, response, log, 41 | corsHeaders)); 42 | } 43 | 44 | // POST complete multipart upload 45 | if (query.uploadId !== undefined) { 46 | return api.callApiMethod('completeMultipartUpload', request, 47 | response, log, (err, result, resHeaders) => 48 | routesUtils.responseXMLBody(err, result, response, log, 49 | resHeaders)); 50 | } 51 | 52 | // POST multiObjectDelete 53 | if (query.delete !== undefined) { 54 | return api.callApiMethod('multiObjectDelete', request, response, 55 | log, (err, xml, corsHeaders) => 56 | routesUtils.responseXMLBody(err, xml, response, log, 57 | corsHeaders)); 58 | } 59 | 60 | // POST Object restore 61 | if (query.restore !== undefined) { 62 | return api.callApiMethod('objectRestore', request, response, 63 | log, (err, statusCode, resHeaders) => 64 | routesUtils.responseNoBody(err, resHeaders, response, 65 | statusCode, log)); 66 | } 67 | 68 | return routesUtils.responseNoBody(errors.NotImplemented, null, response, 69 | 200, log); 70 | } 71 | -------------------------------------------------------------------------------- /lib/shuffle.ts: -------------------------------------------------------------------------------- 1 | import * as crypto from 'crypto'; 2 | 3 | /** 4 | * Takes an array and shuffles it in place. It does not create a new array. 5 | * @param array the array to shuffle 6 | * @returns the reference on the array 7 | */ 8 | export default function shuffle(array: T[]) { 9 | for (let i = array.length - 1; i > 0; i--) { 10 | const randIndex = crypto.randomInt(0, i); 11 | [array[randIndex], array[i]] = [array[i], array[randIndex]]; 12 | } 13 | return array; 14 | } 15 | -------------------------------------------------------------------------------- /lib/simple-glob.d.ts: -------------------------------------------------------------------------------- 1 | // This module declare the interface for simple-glob. 2 | // simple-glob should probably be discarded in favor of node-glob. 3 | // node-glob is an up to date glob implementation, with support for sync and 4 | // async, and well maintained by the community. 5 | // node-glob is performance oriented and is a little lighter than simple-glob. 6 | declare module 'simple-glob' { 7 | export default function (pattern: string | string[]): string[]; 8 | } 9 | -------------------------------------------------------------------------------- /lib/storage/data/external/GCP/GcpApis/abortMPU.js: -------------------------------------------------------------------------------- 1 | const errorInstances = require('../../../../../errors').errorInstances; 2 | const MpuHelper = require('./mpuHelper'); 3 | const { createMpuKey, logger } = require('../GcpUtils'); 4 | const { logHelper } = require('../../utils'); 5 | 6 | /** 7 | * abortMPU - remove all objects of a GCP Multipart Upload 8 | * @param {object} params - abortMPU params 9 | * @param {string} params.Bucket - bucket name 10 | * @param {string} params.MPU - mpu bucket name 11 | * @param {string} params.Key - object key 12 | * @param {number} params.UploadId - MPU upload id 13 | * @param {function} callback - callback function to call 14 | * @return {undefined} 15 | */ 16 | function abortMPU(params, callback) { 17 | if (!params || !params.Key || !params.UploadId || 18 | !params.Bucket || !params.MPU) { 19 | const error = errorInstances.InvalidRequest 20 | .customizeDescription('Missing required parameter'); 21 | logHelper(logger, 'error', 'error in abortMultipartUpload', error); 22 | return callback(error); 23 | } 24 | const mpuHelper = new MpuHelper(this); 25 | const delParams = { 26 | Bucket: params.Bucket, 27 | MPU: params.MPU, 28 | Prefix: createMpuKey(params.Key, params.UploadId), 29 | }; 30 | return mpuHelper.removeParts(delParams, callback); 31 | } 32 | 33 | module.exports = abortMPU; 34 | -------------------------------------------------------------------------------- /lib/storage/data/external/GCP/GcpApis/createMPU.js: -------------------------------------------------------------------------------- 1 | const { v4: uuid } = require('uuid'); 2 | const errors = require('../../../../../errors').default; 3 | const errorInstances = require('../../../../../errors').errorInstances; 4 | const { logHelper } = require('../../utils'); 5 | 6 | /** 7 | * createMPU - creates a MPU upload on GCP (sets a 0-byte object placeholder 8 | * with for the final composed object) 9 | * @param {object} params - createMPU param 10 | * @param {string} params.Bucket - bucket name 11 | * @param {string} params.Key - object key 12 | * @param {string} params.Metadata - object Metadata 13 | * @param {string} params.ContentType - Content-Type header 14 | * @param {string} params.CacheControl - Cache-Control header 15 | * @param {string} params.ContentDisposition - Content-Disposition header 16 | * @param {string} params.ContentEncoding - Content-Encoding header 17 | * @param {function} callback - callback function to call with the generated 18 | * upload-id for MPU operations 19 | * @return {undefined} 20 | */ 21 | function createMPU(params, callback) { 22 | // As google cloud does not have a create MPU function, 23 | // create an empty 'init' object that will temporarily store the 24 | // object metadata and return an upload ID to mimic an AWS MPU 25 | if (!params || !params.Bucket || !params.Key) { 26 | const error = errorInstances.InvalidRequest 27 | .customizeDescription('Missing required parameter'); 28 | logHelper(logger, 'error', 'error in createMultipartUpload', error); 29 | return callback(error); 30 | } 31 | const uploadId = uuid().replace(/-/g, ''); 32 | const mpuParams = { 33 | Bucket: params.Bucket, 34 | Key: createMpuKey(params.Key, uploadId, 'init'), 35 | Metadata: params.Metadata, 36 | ContentType: params.ContentType, 37 | CacheControl: params.CacheControl, 38 | ContentDisposition: params.ContentDisposition, 39 | ContentEncoding: params.ContentEncoding, 40 | }; 41 | mpuParams.Metadata = getPutTagsMetadata(mpuParams.Metadata, params.Tagging); 42 | return this.putObject(mpuParams, err => { 43 | if (err) { 44 | logHelper(logger, 'error', 'error in createMPU - putObject', err); 45 | return callback(err); 46 | } 47 | return callback(null, { UploadId: uploadId }); 48 | }); 49 | } 50 | 51 | module.exports = createMPU; 52 | -------------------------------------------------------------------------------- /lib/storage/data/external/GCP/GcpApis/deleteTagging.js: -------------------------------------------------------------------------------- 1 | const async = require('async'); 2 | 3 | const { stripTags } = require('../GcpUtils'); 4 | 5 | function deleteObjectTagging(params, callback) { 6 | return async.waterfall([ 7 | next => this.headObject({ 8 | Bucket: params.Bucket, 9 | Key: params.Key, 10 | VersionId: params.VersionId, 11 | }, next), 12 | (resObj, next) => { 13 | const completeMD = stripTags(resObj.Metadata); 14 | this.copyObject({ 15 | Bucket: params.Bucket, 16 | Key: params.Key, 17 | CopySource: `${params.Bucket}/${params.Key}`, 18 | Metadata: completeMD, 19 | MetadataDirective: 'REPLACE', 20 | }, next); 21 | }, 22 | ], callback); 23 | } 24 | module.exports = deleteObjectTagging; 25 | -------------------------------------------------------------------------------- /lib/storage/data/external/GCP/GcpApis/getTagging.js: -------------------------------------------------------------------------------- 1 | const { retrieveTags } = require('../GcpUtils'); 2 | 3 | function getObjectTagging(params, callback) { 4 | const headParams = { 5 | Bucket: params.Bucket, 6 | Key: params.Key, 7 | VersionId: params.VersionId, 8 | }; 9 | this.headObject(headParams, (err, res) => { 10 | const TagSet = retrieveTags(res.Metadata); 11 | const retObj = { 12 | VersionId: res.VersionId, 13 | TagSet, 14 | }; 15 | return callback(null, retObj); 16 | }); 17 | } 18 | 19 | module.exports = getObjectTagging; 20 | -------------------------------------------------------------------------------- /lib/storage/data/external/GCP/GcpApis/index.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | // mpu functions 3 | abortMultipartUpload: require('./abortMPU'), 4 | completeMultipartUpload: require('./completeMPU'), 5 | createMultipartUpload: require('./createMPU'), 6 | listParts: require('./listParts'), 7 | uploadPart: require('./uploadPart'), 8 | uploadPartCopy: require('./uploadPartCopy'), 9 | // object tagging 10 | putObject: require('./putObject'), 11 | putObjectTagging: require('./putTagging'), 12 | getObjectTagging: require('./getTagging'), 13 | deleteObjectTagging: require('./deleteTagging'), 14 | }; 15 | -------------------------------------------------------------------------------- /lib/storage/data/external/GCP/GcpApis/listParts.js: -------------------------------------------------------------------------------- 1 | const errorInstances = require('../../../../../errors').errorInstances; 2 | const { createMpuKey, logger } = require('../GcpUtils'); 3 | const { logHelper } = require('../../utils'); 4 | 5 | /** 6 | * listParts - list uploaded MPU parts 7 | * @param {object} params - listParts param 8 | * @param {string} params.Bucket - bucket name 9 | * @param {string} params.Key - object key 10 | * @param {string} params.UploadId - MPU upload id 11 | * @param {function} callback - callback function to call with the list of parts 12 | * @return {undefined} 13 | */ 14 | function listParts(params, callback) { 15 | if (!params || !params.UploadId || !params.Bucket || !params.Key) { 16 | const error = errorInstances.InvalidRequest 17 | .customizeDescription('Missing required parameter'); 18 | logHelper(logger, 'error', 'error in listParts', error); 19 | return callback(error); 20 | } 21 | if (params.PartNumberMarker && params.PartNumberMarker < 0) { 22 | return callback(errorInstances.InvalidArgument 23 | .customizeDescription('The request specified an invalid marker')); 24 | } 25 | const mpuParams = { 26 | Bucket: params.Bucket, 27 | Prefix: createMpuKey(params.Key, params.UploadId, 'parts'), 28 | Marker: createMpuKey(params.Key, params.UploadId, 29 | params.PartNumberMarker, 'parts'), 30 | MaxKeys: params.MaxParts, 31 | }; 32 | return this.listObjects(mpuParams, (err, res) => { 33 | if (err) { 34 | logHelper(logger, 'error', 35 | 'error in listParts - listObjects', err); 36 | return callback(err); 37 | } 38 | return callback(null, res); 39 | }); 40 | } 41 | 42 | module.exports = listParts; 43 | -------------------------------------------------------------------------------- /lib/storage/data/external/GCP/GcpApis/putObject.js: -------------------------------------------------------------------------------- 1 | const { getPutTagsMetadata } = require('../GcpUtils'); 2 | 3 | function putObject(params, callback) { 4 | const putParams = Object.assign({}, params); 5 | putParams.Metadata = getPutTagsMetadata(putParams.Metadata, params.Tagging); 6 | delete putParams.Tagging; 7 | // error handling will be by the actual putObject request 8 | return this.putObjectReq(putParams, callback); 9 | } 10 | 11 | module.exports = putObject; 12 | -------------------------------------------------------------------------------- /lib/storage/data/external/GCP/GcpApis/putTagging.js: -------------------------------------------------------------------------------- 1 | const async = require('async'); 2 | const errors = require('../../../../../errors').default; 3 | 4 | const { processTagSet } = require('../GcpUtils'); 5 | 6 | function putObjectTagging(params, callback) { 7 | if (!params.Tagging || !params.Tagging.TagSet) { 8 | return callback(errors.MissingParameter); 9 | } 10 | const tagRes = processTagSet(params.Tagging.TagSet); 11 | if (tagRes instanceof Error) { 12 | return callback(tagRes); 13 | } 14 | return async.waterfall([ 15 | next => this.headObject({ 16 | Bucket: params.Bucket, 17 | Key: params.Key, 18 | VersionId: params.VersionId, 19 | }, next), 20 | (resObj, next) => { 21 | const completeMD = Object.assign({}, resObj.Metadata, tagRes); 22 | this.copyObject({ 23 | Bucket: params.Bucket, 24 | Key: params.Key, 25 | CopySource: `${params.Bucket}/${params.Key}`, 26 | Metadata: completeMD, 27 | MetadataDirective: 'REPLACE', 28 | }, next); 29 | }, 30 | ], callback); 31 | } 32 | 33 | module.exports = putObjectTagging; 34 | -------------------------------------------------------------------------------- /lib/storage/data/external/GCP/GcpApis/uploadPart.js: -------------------------------------------------------------------------------- 1 | const errorInstances = require('../../../../../errors').errorInstances; 2 | const { getPartNumber, createMpuKey, logger } = require('../GcpUtils'); 3 | const { logHelper } = require('../../utils'); 4 | 5 | /** 6 | * uploadPart - upload part 7 | * @param {object} params - upload part params 8 | * @param {string} params.Bucket - bucket name 9 | * @param {string} params.Key - object key 10 | * @param {function} callback - callback function to call 11 | * @return {undefined} 12 | */ 13 | function uploadPart(params, callback) { 14 | if (!params || !params.UploadId || !params.Bucket || !params.Key) { 15 | const error = errorInstances.InvalidRequest 16 | .customizeDescription('Missing required parameter'); 17 | logHelper(logger, 'error', 'error in uploadPart', error); 18 | return callback(error); 19 | } 20 | const partNumber = getPartNumber(params.PartNumber); 21 | if (!partNumber) { 22 | const error = errorInstances.InvalidArgument 23 | .customizeDescription('PartNumber is invalid'); 24 | logHelper(logger, 'debug', 'error in uploadPart', error); 25 | return callback(error); 26 | } 27 | const mpuParams = { 28 | Bucket: params.Bucket, 29 | Key: createMpuKey(params.Key, params.UploadId, partNumber), 30 | Body: params.Body, 31 | ContentLength: params.ContentLength, 32 | }; 33 | return this.putObjectReq(mpuParams, (err, res) => { 34 | if (err) { 35 | logHelper(logger, 'error', 36 | 'error in uploadPart - putObject', err); 37 | return callback(err); 38 | } 39 | return callback(null, res); 40 | }); 41 | } 42 | 43 | module.exports = uploadPart; 44 | -------------------------------------------------------------------------------- /lib/storage/data/external/GCP/GcpApis/uploadPartCopy.js: -------------------------------------------------------------------------------- 1 | const errorInstances = require('../../../../../errors').errorInstances; 2 | const { getPartNumber, createMpuKey, logger } = require('../GcpUtils'); 3 | const { logHelper } = require('../../utils'); 4 | 5 | /** 6 | * uploadPartCopy - upload part copy 7 | * @param {object} params - upload part copy params 8 | * @param {string} params.Bucket - bucket name 9 | * @param {string} params.Key - object key 10 | * @param {string} params.CopySource - source object to copy 11 | * @param {function} callback - callback function to call 12 | * @return {undefined} 13 | */ 14 | function uploadPartCopy(params, callback) { 15 | if (!params || !params.UploadId || !params.Bucket || !params.Key || 16 | !params.CopySource) { 17 | const error = errorInstances.InvalidRequest 18 | .customizeDescription('Missing required parameter'); 19 | logHelper(logger, 'error', 'error in uploadPartCopy', error); 20 | return callback(error); 21 | } 22 | const partNumber = getPartNumber(params.PartNumber); 23 | if (!partNumber) { 24 | const error = errorInstances.InvalidArgument 25 | .customizeDescription('PartNumber is not a number'); 26 | logHelper(logger, 'debug', 'error in uploadPartCopy', error); 27 | return callback(error); 28 | } 29 | const mpuParams = { 30 | Bucket: params.Bucket, 31 | Key: createMpuKey(params.Key, params.UploadId, partNumber), 32 | CopySource: params.CopySource, 33 | }; 34 | return this.copyObject(mpuParams, callback); 35 | } 36 | 37 | module.exports = uploadPartCopy; 38 | -------------------------------------------------------------------------------- /lib/storage/data/external/GCP/GcpSigner.js: -------------------------------------------------------------------------------- 1 | const url = require('url'); 2 | const qs = require('querystring'); 3 | const AWS = require('aws-sdk'); 4 | const werelogs = require('werelogs'); 5 | const { constructStringToSignV2 } = require('../../../../auth/auth').client; 6 | 7 | const logger = new werelogs.Logger('GcpSigner'); 8 | 9 | function genQueryObject(uri) { 10 | const queryString = url.parse(uri).query; 11 | return qs.parse(queryString); 12 | } 13 | 14 | const GcpSigner = AWS.util.inherit(AWS.Signers.RequestSigner, { 15 | constructor: function GcpSigner(request) { 16 | AWS.Signers.RequestSigner.call(this, request); 17 | }, 18 | 19 | addAuthorization: function addAuthorization(credentials, date) { 20 | if (!this.request.headers['presigned-expires']) { 21 | this.request.headers['x-goog-date'] = AWS.util.date.rfc822(date); 22 | } 23 | 24 | const signature = 25 | this.sign(credentials.secretAccessKey, this.stringToSign()); 26 | const auth = `GOOG1 ${credentials.accessKeyId}: ${signature}`; 27 | 28 | this.request.headers.Authorization = auth; 29 | }, 30 | 31 | stringToSign: function stringToSign() { 32 | const requestObject = { 33 | url: this.request.path, 34 | method: this.request.method, 35 | host: this.request.endpoint.host, 36 | headers: this.request.headers, 37 | bucketName: this.request.virtualHostedBucket, 38 | query: genQueryObject(this.request.path) || {}, 39 | }; 40 | requestObject.gotBucketNameFromHost = 41 | requestObject.host.indexOf(this.request.virtualHostedBucket) >= 0; 42 | const data = Object.assign({}, this.request.headers); 43 | return constructStringToSignV2(requestObject, data, logger, 'GCP'); 44 | }, 45 | 46 | sign: function sign(secret, string) { 47 | return AWS.util.crypto.hmac(secret, string, 'base64', 'sha1'); 48 | }, 49 | }); 50 | 51 | module.exports = GcpSigner; 52 | -------------------------------------------------------------------------------- /lib/storage/data/external/GCP/index.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | GCP: require('./GcpService'), 3 | GcpSigner: require('./GcpSigner'), 4 | GcpUtils: require('./GcpUtils'), 5 | }; 6 | -------------------------------------------------------------------------------- /lib/storage/data/file/DataFileInterface.js: -------------------------------------------------------------------------------- 1 | const RESTClient = require('../../../network/rest/RESTClient').default; 2 | 3 | class DataFileInterface { 4 | constructor(config) { 5 | const { host, port } = config.dataClient; 6 | 7 | this.restClient = new RESTClient( 8 | { host, port }); 9 | } 10 | 11 | toObjectGetInfo(objectKey) { 12 | return { key: objectKey }; 13 | } 14 | 15 | put(stream, size, keyContext, reqUids, callback) { 16 | // ignore keyContext 17 | this.restClient.put(stream, size, reqUids, callback); 18 | } 19 | 20 | get(objectGetInfo, range, reqUids, callback) { 21 | const key = objectGetInfo.key ? objectGetInfo.key : objectGetInfo; 22 | this.restClient.get(key, range, reqUids, callback); 23 | } 24 | 25 | delete(objectGetInfo, reqUids, callback) { 26 | const key = objectGetInfo.key ? objectGetInfo.key : objectGetInfo; 27 | this.restClient.delete(key, reqUids, callback); 28 | } 29 | 30 | getDiskUsage(config, reqUids, callback) { 31 | this.restClient.getAction('diskUsage', reqUids, (err, val) => { 32 | if (err) { 33 | return callback(err); 34 | } 35 | return callback(null, JSON.parse(val)); 36 | }); 37 | } 38 | } 39 | 40 | module.exports = DataFileInterface; 41 | -------------------------------------------------------------------------------- /lib/storage/data/file/utils.js: -------------------------------------------------------------------------------- 1 | const posixFadvise = require('fcntl'); 2 | 3 | /** 4 | * Release free cached pages associated with a file 5 | * 6 | * @param {String} filePath - absolute path of the associated file 7 | * @param {Int} fd - file descriptor of the associated file 8 | * @param {werelogs.RequestLogger} log - logging object 9 | * @return {undefined} 10 | */ 11 | function releasePageCacheSync(filePath, fd, log) { 12 | const ret = posixFadvise(fd, 0, 0, 4); 13 | if (ret !== 0) { 14 | log.warning( 15 | `error fadv_dontneed ${filePath} returned ${ret}`); 16 | } 17 | } 18 | 19 | module.exports = releasePageCacheSync; 20 | -------------------------------------------------------------------------------- /lib/storage/data/utils/RelayMD5Sum.js: -------------------------------------------------------------------------------- 1 | const crypto = require('crypto'); 2 | const { Transform } = require('stream'); 3 | 4 | /** 5 | * This class is designed to compute the md5 hash on multiple streams in relay 6 | * race style. 7 | */ 8 | class RelayMD5Sum extends Transform { 9 | /** 10 | * @constructor 11 | * @param {Hash} starterHash - hash from prior stream (if any) to be updated 12 | * with new stream here 13 | * @param {function} done Callback - This callback is called when 14 | * the hash computation is finished, this function need to be synchronous 15 | * and being call before the end of the stream object 16 | */ 17 | constructor(starterHash, done) { 18 | super({}); 19 | this.hash = starterHash || crypto.createHash('md5'); 20 | this.done = done; 21 | } 22 | 23 | /** 24 | * This function will update the current md5 hash with the next chunk 25 | * 26 | * @param {Buffer|string} chunk - Chunk to compute 27 | * @param {string} encoding - Data encoding 28 | * @param {function} callback - Callback(err, chunk, encoding) 29 | * @return {undefined} 30 | */ 31 | _transform(chunk, encoding, callback) { 32 | this.hash.update(chunk, encoding); 33 | this.done(this.hash); 34 | callback(null, chunk, encoding); 35 | } 36 | } 37 | 38 | module.exports = RelayMD5Sum; 39 | -------------------------------------------------------------------------------- /lib/storage/metadata/conditions.js: -------------------------------------------------------------------------------- 1 | const joi = require('joi'); 2 | 3 | const supportedOperators = { 4 | $eq: true, 5 | $ne: true, 6 | $gt: true, 7 | $gte: true, 8 | $lt: true, 9 | $lte: true, 10 | }; 11 | 12 | // supports strings and numbers 13 | const _operatorType1 = joi.string().valid( 14 | '$gt', 15 | '$gte', 16 | '$lt', 17 | '$lte', 18 | ); 19 | 20 | // supports strings, numbers, and boolean 21 | const _operatorType2 = joi.string().valid( 22 | '$eq', 23 | '$ne', 24 | ); 25 | 26 | const _valueType1 = joi.alternatives([ 27 | joi.string(), 28 | joi.number(), 29 | ]); 30 | 31 | const _valueType2 = joi.alternatives([ 32 | joi.string(), 33 | joi.number(), 34 | joi.boolean(), 35 | ]); 36 | 37 | const queryObject = joi.object({}) 38 | .pattern(_operatorType1, _valueType1) 39 | .pattern(_operatorType2, _valueType2) 40 | .xor(...Object.keys(supportedOperators)); 41 | 42 | const metadataCondObject = joi.alternatives([ 43 | _valueType1, 44 | _valueType2, 45 | queryObject, 46 | ]); 47 | 48 | function validateConditionsObject(obj) { 49 | if (obj === undefined) { 50 | return false; 51 | } 52 | const res = metadataCondObject.validate(obj); 53 | if (res.error) { 54 | return false; 55 | } 56 | return true; 57 | } 58 | 59 | module.exports = { 60 | supportedOperators, 61 | validateConditionsObject, 62 | }; 63 | -------------------------------------------------------------------------------- /lib/storage/metadata/in_memory/ListMultipartUploadsResult.js: -------------------------------------------------------------------------------- 1 | const ListResult = require('./ListResult'); 2 | 3 | class ListMultipartUploadsResult extends ListResult { 4 | constructor() { 5 | super(); 6 | this.Uploads = []; 7 | this.NextKeyMarker = undefined; 8 | this.NextUploadIdMarker = undefined; 9 | } 10 | 11 | addUpload(uploadInfo) { 12 | this.Uploads.push({ 13 | key: decodeURIComponent(uploadInfo.key), 14 | value: { 15 | UploadId: uploadInfo.uploadId, 16 | Initiator: { 17 | ID: uploadInfo.initiatorID, 18 | DisplayName: uploadInfo.initiatorDisplayName, 19 | }, 20 | Owner: { 21 | ID: uploadInfo.ownerID, 22 | DisplayName: uploadInfo.ownerDisplayName, 23 | }, 24 | StorageClass: uploadInfo.storageClass, 25 | Initiated: uploadInfo.initiated, 26 | }, 27 | }); 28 | this.MaxKeys += 1; 29 | } 30 | } 31 | 32 | module.exports = ListMultipartUploadsResult; 33 | -------------------------------------------------------------------------------- /lib/storage/metadata/in_memory/ListResult.js: -------------------------------------------------------------------------------- 1 | class ListResult { 2 | constructor() { 3 | this.IsTruncated = false; 4 | this.NextMarker = undefined; 5 | this.CommonPrefixes = []; 6 | /* 7 | Note: this.MaxKeys will get incremented as 8 | keys are added so that when response is returned, 9 | this.MaxKeys will equal total keys in response 10 | (with each CommonPrefix counting as 1 key) 11 | */ 12 | this.MaxKeys = 0; 13 | } 14 | 15 | addCommonPrefix(prefix) { 16 | if (!this.hasCommonPrefix(prefix)) { 17 | this.CommonPrefixes.push(prefix); 18 | this.MaxKeys += 1; 19 | } 20 | } 21 | 22 | hasCommonPrefix(prefix) { 23 | return (this.CommonPrefixes.indexOf(prefix) !== -1); 24 | } 25 | } 26 | 27 | module.exports = ListResult; 28 | -------------------------------------------------------------------------------- /lib/storage/metadata/in_memory/bucket_utilities.js: -------------------------------------------------------------------------------- 1 | function markerFilterMPU(allMarkers, array) { 2 | const { keyMarker, uploadIdMarker } = allMarkers; 3 | 4 | // 1. if the item key matches the keyMarker and an uploadIdMarker exists, 5 | // find the first uploadId in the array that is alphabetically after 6 | // uploadIdMarker 7 | // 2. if the item key does not match the keyMarker, find the first uploadId 8 | // in the array that is alphabetically after keyMarker 9 | const firstUnfilteredIndex = array.findIndex( 10 | item => (uploadIdMarker && item.key === keyMarker ? 11 | item.uploadId > uploadIdMarker : 12 | item.key > keyMarker)); 13 | return firstUnfilteredIndex !== -1 ? array.slice(firstUnfilteredIndex) : []; 14 | } 15 | 16 | function prefixFilter(prefix, array) { 17 | for (let i = 0; i < array.length; i++) { 18 | if (array[i].indexOf(prefix) !== 0) { 19 | array.splice(i, 1); 20 | i--; 21 | } 22 | } 23 | return array; 24 | } 25 | 26 | function isKeyInContents(responseObject, key) { 27 | return responseObject.Contents.some(val => val.key === key); 28 | } 29 | 30 | module.exports = { 31 | markerFilterMPU, 32 | prefixFilter, 33 | isKeyInContents, 34 | }; 35 | -------------------------------------------------------------------------------- /lib/storage/metadata/in_memory/metadata.js: -------------------------------------------------------------------------------- 1 | const metadata = { 2 | buckets: new Map, 3 | keyMaps: new Map, 4 | }; 5 | 6 | module.exports = { 7 | metadata, 8 | }; 9 | -------------------------------------------------------------------------------- /lib/storage/metadata/proxy/README.md: -------------------------------------------------------------------------------- 1 | # Metatada Proxy Server 2 | 3 | ## Design goals 4 | 5 | ## Design choices 6 | 7 | ## Implementation details 8 | 9 | ## How to run the proxy server 10 | 11 | ```js 12 | const werelogs = require('werelogs'); 13 | const MetadataWrapper = require('arsenal') 14 | .storage.metadata.MetadataWrapper; 15 | const Server = require('arsenal') 16 | .storage.metadata.proxy.Server; 17 | 18 | const logger = new werelogs.Logger('MetadataProxyServer', 19 | 'debug', 'debug'); 20 | const metadataWrapper = new MetadataWrapper('mem', {}, 21 | null, logger); 22 | const server = new Server(metadataWrapper, 23 | { 24 | port: 9001, 25 | workers: 1, 26 | }, 27 | logger); 28 | server.start(() => { 29 | logger.info('Metadata Proxy Server successfully started. ' + 30 | `Using the ${metadataWrapper.implName} backend`); 31 | }); 32 | ``` 33 | -------------------------------------------------------------------------------- /lib/stream/index.ts: -------------------------------------------------------------------------------- 1 | export { default as readJSONStreamObject } from './readJSONStreamObject'; 2 | -------------------------------------------------------------------------------- /lib/stream/readJSONStreamObject.ts: -------------------------------------------------------------------------------- 1 | import { errorInstances } from '../errors'; 2 | import * as stream from 'stream'; 3 | import joi from 'joi'; 4 | 5 | /** 6 | * read a JSON object from a stream returned as a javascript object, 7 | * handle errors. 8 | * 9 | * @param s - Readable stream 10 | * @param [joiSchema] - optional validation schema for the JSON object 11 | * @return a Promise resolved with the parsed JSON object as a result 12 | */ 13 | export default async function readJSONStreamObject( 14 | s: stream.Readable, 15 | joiSchema?: joi.Schema 16 | ): Promise { 17 | return new Promise((resolve, reject) => { 18 | const contentsChunks: any = []; 19 | s.on('data', chunk => contentsChunks.push(chunk)); 20 | s.on('end', () => { 21 | const contents = contentsChunks.join(''); 22 | try { 23 | const parsedContents = JSON.parse(contents); 24 | if (joiSchema) { 25 | const { error, value } = joiSchema.validate(parsedContents); 26 | if (error) { 27 | throw error; 28 | } 29 | return resolve(value); 30 | } 31 | return resolve(parsedContents); 32 | } catch (err: any) { 33 | return reject( 34 | errorInstances.InvalidArgument.customizeDescription( 35 | `invalid input: ${err.message}` 36 | ) 37 | ); 38 | } 39 | }); 40 | s.once('error', reject); 41 | }); 42 | } 43 | -------------------------------------------------------------------------------- /lib/stringHash.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * This function compute a hash from a string 3 | * https://github.com/darkskyapp/string-hash/blob/master/index.js 4 | * @param str - The string to compute the hash 5 | * @return The computed hash 6 | */ 7 | export default function stringHash(str: string): number { 8 | let hash = 5381; 9 | let i = str.length; 10 | 11 | while (i) { 12 | hash = hash * 33 ^ str.charCodeAt(--i); 13 | } 14 | 15 | /* JavaScript does bitwise operations (like XOR, above) on 16 | * 32-bit signed integers. Since we want the results to be 17 | * always positive, convert the signed int to an unsigned by 18 | * doing an unsigned bitshift. 19 | */ 20 | return hash >>> 0; 21 | } 22 | -------------------------------------------------------------------------------- /lib/types.ts: -------------------------------------------------------------------------------- 1 | import { ArsenalError } from './errors'; 2 | 3 | export interface ArsenalCallback { 4 | (err: null, result: T): void; 5 | (err: void): void; 6 | (err: ArsenalError): void; 7 | }; 8 | 9 | type Primitive = string | number | boolean | symbol | null | undefined | bigint; 10 | 11 | type StripKey = 12 | K extends `${Exclude}.${infer R}` ? R : never; 13 | 14 | export type NestedOmit = 15 | T extends Primitive | Function 16 | ? T 17 | : T extends Array 18 | ? Array> 19 | : { 20 | [P in keyof T as P extends Extract ? never : P]: 21 | NestedOmit>; 22 | }; 23 | -------------------------------------------------------------------------------- /lib/versioning/constants.ts: -------------------------------------------------------------------------------- 1 | export enum BucketVersioningFormat { 2 | V0 = 'v0', 3 | V0MIG = 'v0mig', 4 | V0V1 = 'v0v1', 5 | V1MIG = 'v1mig', 6 | V1 = 'v1', 7 | }; 8 | 9 | export const VersioningConstants = { 10 | VersionId: { 11 | Separator: '\0', 12 | }, 13 | DbPrefixes: { 14 | Master: '\x7fM', 15 | Version: '\x7fV', 16 | Replay: '\x7fR', 17 | }, 18 | BucketVersioningKeyFormat: { 19 | current: BucketVersioningFormat.V1, 20 | v0: BucketVersioningFormat.V0, 21 | v0mig: BucketVersioningFormat.V0MIG, 22 | v0v1: BucketVersioningFormat.V0V1, 23 | v1mig: BucketVersioningFormat.V1MIG, 24 | v1: BucketVersioningFormat.V1, 25 | }, 26 | ExternalNullVersionId: 'null', 27 | }; 28 | -------------------------------------------------------------------------------- /lib/versioning/index.ts: -------------------------------------------------------------------------------- 1 | export { VersioningConstants } from './constants'; 2 | export { Version, isMasterKey } from './Version'; 3 | export * as VersionID from './VersionID'; 4 | export { default as WriteGatheringManager } from './WriteGatheringManager'; 5 | export { default as WriteCache } from './WriteCache'; 6 | export { default as VersioningRequestProcessor } from './VersioningRequestProcessor'; 7 | -------------------------------------------------------------------------------- /tests/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "jest": true 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /tests/functional/clustering/clustering.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict'; // eslint-disable-line 2 | 3 | const spawn = require('child_process').spawn; 4 | 5 | let currentSpawn = undefined; 6 | 7 | function runTest(name, done) { 8 | const test = spawn('yarn', ['ts-node', '--transpile-only', `${__dirname}/utils/${name}.js`]); 9 | currentSpawn = test; 10 | test.stdout.pipe(process.stdout); 11 | test.stderr.pipe(process.stderr); 12 | test.on('close', code => { 13 | currentSpawn = undefined; 14 | if (!code) { 15 | return done(); 16 | } 17 | return done(new Error(`test '${name}' failed with code ${code}`)); 18 | }); 19 | } 20 | 21 | describe('Clustering', () => { 22 | afterEach(done => { 23 | if (currentSpawn) { 24 | const safeTimeout = setTimeout(() => { 25 | currentSpawn.kill('SIGKILL'); 26 | done(); 27 | }, 5000); 28 | currentSpawn.removeAllListeners(); 29 | currentSpawn.on('close', () => { 30 | clearTimeout(safeTimeout); 31 | done(); 32 | }); 33 | return currentSpawn.kill('SIGTERM'); 34 | } 35 | return done(); 36 | }); 37 | 38 | it('Should create and stop workers properly', done => { 39 | runTest('simple', done); 40 | }); 41 | 42 | it('Should restart workers until clustering stopped', done => { 43 | runTest('watchdog', done); 44 | }); 45 | 46 | it('Should shutdown cluster if master killed', done => { 47 | runTest('killed', done); 48 | }); 49 | 50 | it('Should timeout shutdown of workers if not exiting properly', done => { 51 | runTest('shutdownTimeout', done); 52 | }); 53 | }); 54 | -------------------------------------------------------------------------------- /tests/functional/clustering/utils/killed.js: -------------------------------------------------------------------------------- 1 | 'use strict'; // eslint-disable-line 2 | 3 | const Logger = require('werelogs').Logger; 4 | const http = require('http'); 5 | const Clustering = require('../../../../lib/Clustering').default; 6 | const Cluster = require('cluster'); 7 | 8 | const log = new Logger('S3', { 9 | level: 'trace', 10 | dump: 'warn', 11 | }); 12 | 13 | const ErrorCodes = { 14 | noError: 0, 15 | WorkerNotRestarted: 1, 16 | WorkerNotExited: 2, 17 | }; 18 | let result = ErrorCodes.noError; 19 | 20 | const clusters = new Clustering(4, log); 21 | clusters.start(current => { 22 | http.createServer(req => { 23 | if (req.url === '/crash') { 24 | throw new Error('Wanted crash'); 25 | } 26 | }).listen(14000, () => { 27 | log.info('listening', { 28 | id: current.getIndex(), 29 | }); 30 | }); 31 | }).onExit(current => { 32 | if (current.isMaster()) { 33 | return setTimeout(() => { 34 | if (result !== ErrorCodes.noError) { 35 | return process.exit(result); 36 | } 37 | current.getWorkers().forEach(worker => { 38 | if (worker) { 39 | result = ErrorCodes.WorkerNotExited; 40 | } 41 | }); 42 | return process.exit(result); 43 | }, 500); 44 | } 45 | log.info('exiting', { 46 | id: current.getIndex(), 47 | }); 48 | return process.exit(0); 49 | }); 50 | 51 | if (Cluster.isMaster) { 52 | let kill = true; 53 | const watchdog = setInterval(() => { 54 | if (kill) { 55 | kill = false; 56 | const req = http.get({ 57 | path: '/crash', 58 | port: 14000, 59 | }, () => {}); 60 | req.on('error', () => {}); 61 | return undefined; 62 | } 63 | kill = true; 64 | return clusters.getWorkers().forEach((worker, i) => { 65 | if (!worker) { 66 | log.error('Worker not restarted', { 67 | id: i, 68 | }); 69 | result = ErrorCodes.WorkerNotRestarted; 70 | return undefined; 71 | } 72 | try { 73 | return process.kill(worker.process.pid, 0); 74 | } catch (e) { 75 | log.error('Worker not restarted', { 76 | id: i, 77 | }); 78 | result = ErrorCodes.WorkerNotRestarted; 79 | return undefined; 80 | } 81 | }); 82 | }, 500); 83 | setTimeout(() => { 84 | clearInterval(watchdog); 85 | process.kill(process.pid, 'SIGINT'); 86 | }, 5000); 87 | } 88 | -------------------------------------------------------------------------------- /tests/functional/clustering/utils/shutdownTimeout.js: -------------------------------------------------------------------------------- 1 | 'use strict'; // eslint-disable-line 2 | 3 | const Logger = require('werelogs').Logger; 4 | const http = require('http'); 5 | const Clustering = require('../../../../lib/Clustering').default; 6 | const Cluster = require('cluster'); 7 | 8 | const log = new Logger('S3', { 9 | level: 'trace', 10 | dump: 'warn', 11 | }); 12 | 13 | const ErrorCodes = { 14 | noError: 0, 15 | WorkerNotStarted: 1, 16 | WorkerNotExited: 2, 17 | }; 18 | let result = ErrorCodes.noError; 19 | 20 | const clusters = new Clustering(4, log); 21 | clusters.start(current => { 22 | http.createServer(() => {}).listen(14000, () => { 23 | log.info('listening', { 24 | id: current.getIndex(), 25 | }); 26 | }); 27 | }).onExit((current, signal) => { 28 | if (current.isMaster()) { 29 | return setTimeout(() => { 30 | if (result !== ErrorCodes.noError) { 31 | return process.exit(result); 32 | } 33 | current.getWorkers().forEach(worker => { 34 | if (worker) { 35 | result = ErrorCodes.WorkerNotExited; 36 | } 37 | }); 38 | return process.exit(result); 39 | }, 500); 40 | } 41 | if (signal === 'SIGTERM') { 42 | log.info('silent exiting', { 43 | id: current.getIndex(), 44 | }); 45 | return undefined; 46 | } 47 | return process.exit(0); 48 | }); 49 | 50 | if (Cluster.isMaster) { 51 | setTimeout(() => { 52 | clusters.getWorkers().forEach((worker, i) => { 53 | if (!worker) { 54 | log.error('Worker not started', { 55 | id: i, 56 | }); 57 | result = ErrorCodes.WorkerNotStarted; 58 | return undefined; 59 | } 60 | try { 61 | return process.kill(worker.process.pid, 0); 62 | } catch (e) { 63 | log.error('Worker not started', { 64 | id: i, 65 | }); 66 | result = ErrorCodes.WorkerNotStarted; 67 | return undefined; 68 | } 69 | }); 70 | clusters.stop(); 71 | }, 500); 72 | } 73 | -------------------------------------------------------------------------------- /tests/functional/clustering/utils/simple.js: -------------------------------------------------------------------------------- 1 | 'use strict'; // eslint-disable-line 2 | 3 | const Logger = require('werelogs').Logger; 4 | const http = require('http'); 5 | const Clustering = require('../../../../lib/Clustering').default; 6 | const Cluster = require('cluster'); 7 | 8 | const log = new Logger('S3', { 9 | level: 'trace', 10 | dump: 'warn', 11 | }); 12 | 13 | const ErrorCodes = { 14 | noError: 0, 15 | WorkerNotStarted: 1, 16 | WorkerNotExited: 2, 17 | }; 18 | let result = ErrorCodes.noError; 19 | 20 | const clusters = new Clustering(4, log); 21 | clusters.start(current => { 22 | http.createServer(() => {}).listen(14000, () => { 23 | log.info('listening', { 24 | id: current.getIndex(), 25 | }); 26 | }); 27 | }).onExit(current => { 28 | if (current.isMaster()) { 29 | return setTimeout(() => { 30 | if (result !== ErrorCodes.noError) { 31 | return process.exit(result); 32 | } 33 | current.getWorkers().forEach(worker => { 34 | if (worker) { 35 | result = ErrorCodes.WorkerNotExited; 36 | } 37 | }); 38 | return process.exit(result); 39 | }, 500); 40 | } 41 | log.info('exiting', { 42 | id: current.getIndex(), 43 | }); 44 | return process.exit(0); 45 | }); 46 | 47 | if (Cluster.isMaster) { 48 | setTimeout(() => { 49 | clusters.getWorkers().forEach((worker, i) => { 50 | if (!worker) { 51 | log.error('Worker not started', { 52 | id: i, 53 | }); 54 | result = ErrorCodes.WorkerNotStarted; 55 | return undefined; 56 | } 57 | try { 58 | return process.kill(worker.process.pid, 0); 59 | } catch (e) { 60 | log.error('Worker not started', { 61 | id: i, 62 | }); 63 | result = ErrorCodes.WorkerNotStarted; 64 | return undefined; 65 | } 66 | }); 67 | clusters.stop(); 68 | }, 500); 69 | } 70 | -------------------------------------------------------------------------------- /tests/functional/clustering/utils/watchdog.js: -------------------------------------------------------------------------------- 1 | 'use strict'; // eslint-disable-line 2 | 3 | const Logger = require('werelogs').Logger; 4 | const http = require('http'); 5 | const Clustering = require('../../../../lib/Clustering').default; 6 | const Cluster = require('cluster'); 7 | 8 | const log = new Logger('S3', { 9 | level: 'trace', 10 | dump: 'warn', 11 | }); 12 | 13 | const ErrorCodes = { 14 | noError: 0, 15 | WorkerNotRestarted: 1, 16 | WorkerNotExited: 2, 17 | }; 18 | let result = ErrorCodes.noError; 19 | 20 | const clusters = new Clustering(4, log); 21 | clusters.start(current => { 22 | http.createServer(req => { 23 | if (req.url === '/crash') { 24 | throw new Error('Wanted crash'); 25 | } 26 | }).listen(14000, () => { 27 | log.info('listening', { 28 | id: current.getIndex(), 29 | }); 30 | }); 31 | }).onExit(current => { 32 | if (current.isMaster()) { 33 | return setTimeout(() => { 34 | if (result !== ErrorCodes.noError) { 35 | return process.exit(result); 36 | } 37 | current.getWorkers().forEach(worker => { 38 | if (worker) { 39 | result = ErrorCodes.WorkerNotExited; 40 | } 41 | }); 42 | return process.exit(result); 43 | }, 500); 44 | } 45 | log.info('exiting', { 46 | id: current.getIndex(), 47 | }); 48 | return process.exit(0); 49 | }); 50 | 51 | if (Cluster.isMaster) { 52 | let kill = true; 53 | let count = 0; 54 | const watchdog = setInterval(() => { 55 | if (kill) { 56 | ++count; 57 | kill = false; 58 | const req = http.get({ 59 | path: '/crash', 60 | port: 14000, 61 | }, () => {}); 62 | req.on('error', () => {}); 63 | return undefined; 64 | } 65 | kill = true; 66 | clusters.getWorkers().forEach((worker, i) => { 67 | if (!worker) { 68 | result = ErrorCodes.WorkerNotRestarted; 69 | log.error('Worker not started', { 70 | id: i, 71 | }); 72 | return undefined; 73 | } 74 | try { 75 | return process.kill(worker.process.pid, 0); 76 | } catch (e) { 77 | log.error('Worker not restarted', { 78 | id: i, 79 | }); 80 | result = ErrorCodes.WorkerNotRestarted; 81 | return undefined; 82 | } 83 | }); 84 | if (count === 10 || result !== ErrorCodes.noError) { 85 | clearInterval(watchdog); 86 | clusters.stop(); 87 | } 88 | return undefined; 89 | }, 500); 90 | } 91 | -------------------------------------------------------------------------------- /tests/functional/kmip/lowlevel.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict'; // eslint-disable-line strict 2 | 3 | const assert = require('assert'); 4 | const TTLVCodec = require('../../../lib/network/kmip/codec/ttlv').default; 5 | const TransportTemplate = 6 | require('../../../lib/network/kmip/transport/TransportTemplate').default; 7 | const KMIP = require('../../../lib/network/kmip').default; 8 | const { 9 | logger, 10 | MirrorChannel, 11 | } = require('../../utils/kmip/ersatz'); 12 | const lowlevelFixtures = require('../../utils/kmip/lowlevelFixtures'); 13 | 14 | 15 | class MirrorTransport extends TransportTemplate { 16 | constructor(options) { 17 | super(new MirrorChannel(KMIP, TTLVCodec), options); 18 | } 19 | } 20 | 21 | const options = { 22 | kmip: { 23 | codec: {}, 24 | transport: { 25 | pipelineDepth: 8, 26 | tls: { 27 | port: 5696, 28 | }, 29 | }, 30 | }, 31 | }; 32 | 33 | describe('KMIP Low Level Driver', () => { 34 | lowlevelFixtures.forEach((fixture, n) => { 35 | it(`should work with fixture #${n}`, done => { 36 | const kmip = new KMIP(TTLVCodec, MirrorTransport, options); 37 | const requestPayload = fixture.payload(kmip); 38 | kmip.request(logger, fixture.operation, 39 | requestPayload, (err, response) => { 40 | if (err) { 41 | return done(err); 42 | } 43 | const responsePayload = response.lookup( 44 | 'Response Message/Batch Item/Response Payload', 45 | )[0]; 46 | assert.deepStrictEqual(responsePayload, 47 | requestPayload); 48 | return done(); 49 | }); 50 | }); 51 | }); 52 | }); 53 | -------------------------------------------------------------------------------- /tests/functional/kmip/tls.spec.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const net = require('net'); 3 | const tls = require('tls'); 4 | const TransportTemplate = 5 | require('../../../lib/network/kmip/transport/TransportTemplate').default; 6 | const { logger } = require('../../utils/kmip/ersatz'); 7 | 8 | describe('KMIP Connection Management', () => { 9 | let server; 10 | beforeAll(done => { 11 | server = net.createServer(conn => { 12 | // abort the connection as soon as it is accepted 13 | conn.destroy(); 14 | }); 15 | server.listen(5696); 16 | server.on('listening', done); 17 | }); 18 | afterAll(done => { 19 | server.close(done); 20 | }); 21 | 22 | it('should gracefully handle connection errors', done => { 23 | const transport = new TransportTemplate( 24 | tls, 25 | { 26 | pipelineDepth: 1, 27 | tls: { 28 | port: 5696, 29 | }, 30 | }); 31 | const request = Buffer.alloc(10).fill(6); 32 | /* Using a for loop here instead of anything 33 | * asynchronous, the callbacks get stuck in 34 | * the conversation queue and are unwind with 35 | * an error. It is the purpose of this test */ 36 | transport.send(logger, request, (err, conversation, response) => { 37 | assert(err); 38 | assert(!response); 39 | done(); 40 | }); 41 | transport.end(); 42 | }); 43 | }); 44 | -------------------------------------------------------------------------------- /tests/unit/algos/heap/Heap.spec.ts: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import * as Heap from '../../../../lib/algos/heap/Heap'; 4 | 5 | function numberCompare(x: any, y: any): Heap.CompareResult { 6 | if (x > y) { 7 | return Heap.CompareResult.GT; 8 | } 9 | 10 | if (x < y) { 11 | return Heap.CompareResult.LT; 12 | } 13 | 14 | return Heap.CompareResult.EQ; 15 | } 16 | 17 | describe('Heap', () => { 18 | describe('Heap::add', () => { 19 | it('should set max heap size and return error if max size is reached', () => { 20 | const heap = new Heap.Heap(1, Heap.HeapOrder.Min, numberCompare); 21 | expect(heap.add(1)).toBeNull(); 22 | expect(heap.add(2)).not.toBeNull(); 23 | }); 24 | }); 25 | 26 | describe('Heap::remove', () => { 27 | it('should return null if heap is empty', () => { 28 | const heap = new Heap.Heap(1, Heap.HeapOrder.Min, numberCompare); 29 | expect(heap.remove()).toBeNull(); 30 | }); 31 | }); 32 | 33 | describe('Heap::peek', () => { 34 | it('should return null if heap is empty', () => { 35 | const heap = new Heap.Heap(1, Heap.HeapOrder.Min, numberCompare); 36 | expect(heap.peek()).toBeNull(); 37 | }); 38 | }); 39 | }); 40 | 41 | describe('MinHeap', () => { 42 | it('should maintain min heap properties', () => { 43 | const minHeap = new Heap.MinHeap(5, numberCompare); 44 | expect(minHeap.add(5)).toBeNull(); 45 | expect(minHeap.add(3)).toBeNull(); 46 | expect(minHeap.add(2)).toBeNull(); 47 | expect(minHeap.add(4)).toBeNull(); 48 | expect(minHeap.add(1)).toBeNull(); 49 | expect(minHeap.size).toEqual(5); 50 | expect(minHeap.remove()).toEqual(1); 51 | expect(minHeap.remove()).toEqual(2); 52 | expect(minHeap.remove()).toEqual(3); 53 | expect(minHeap.remove()).toEqual(4); 54 | expect(minHeap.remove()).toEqual(5); 55 | expect(minHeap.size).toEqual(0); 56 | }); 57 | }); 58 | 59 | describe('MaxHeap', () => { 60 | it('should maintain max heap properties', () => { 61 | const maxHeap = new Heap.MaxHeap(5, numberCompare); 62 | expect(maxHeap.add(5)).toBeNull(); 63 | expect(maxHeap.add(3)).toBeNull(); 64 | expect(maxHeap.add(2)).toBeNull(); 65 | expect(maxHeap.add(4)).toBeNull(); 66 | expect(maxHeap.add(1)).toBeNull(); 67 | expect(maxHeap.size).toEqual(5); 68 | expect(maxHeap.remove()).toEqual(5); 69 | expect(maxHeap.remove()).toEqual(4); 70 | expect(maxHeap.remove()).toEqual(3); 71 | expect(maxHeap.remove()).toEqual(2); 72 | expect(maxHeap.remove()).toEqual(1); 73 | expect(maxHeap.size).toEqual(0); 74 | }); 75 | }); 76 | -------------------------------------------------------------------------------- /tests/unit/algos/set/SortedSet.spec.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const SortedSet = require('../../../../lib/algos/set/SortedSet'); 3 | 4 | describe('SortedSet', () => { 5 | it('basic', () => { 6 | const set = new SortedSet(); 7 | set.set('foo', 'bar'); 8 | assert(set.isSet('foo')); 9 | assert(!set.isSet('foo2')); 10 | assert.strictEqual(set.get('foo'), 'bar'); 11 | set.set('foo', 'bar2'); 12 | assert.strictEqual(set.get('foo'), 'bar2'); 13 | set.del('foo'); 14 | assert(!set.isSet('foo')); 15 | }); 16 | 17 | it('size', () => { 18 | const set = new SortedSet(); 19 | set.set('foo', 'bar'); 20 | assert.strictEqual(set.size, 1); 21 | set.set('foo2', 'bar'); 22 | assert.strictEqual(set.size, 2); 23 | set.set('foo3', 'bar'); 24 | assert.strictEqual(set.size, 3); 25 | set.del('foo'); 26 | assert.strictEqual(set.size, 2); 27 | set.del('foo2'); 28 | assert.strictEqual(set.size, 1); 29 | set.del('foo3'); 30 | assert.strictEqual(set.size, 0); 31 | }); 32 | }); 33 | -------------------------------------------------------------------------------- /tests/unit/auth/in_memory/indexer.spec.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | 3 | const Indexer = require('../../../../lib/auth/backends/in_memory/Indexer').default; 4 | const ref = require('./sample_authdata.json'); 5 | const { should } = require('./AuthLoader.spec'); 6 | 7 | describe('S3 AuthData Indexer', () => { 8 | let obj = {}; 9 | let index = undefined; 10 | 11 | beforeEach(done => { 12 | obj = JSON.parse(JSON.stringify(ref)); 13 | index = new Indexer(obj); 14 | done(); 15 | }); 16 | 17 | it('Should return account from canonicalID', done => { 18 | const res = index.getEntityByCanId(obj.accounts[0].canonicalID); 19 | assert.strictEqual(typeof res, 'object'); 20 | assert.strictEqual(res.arn, obj.accounts[0].arn); 21 | done(); 22 | }); 23 | 24 | it('Should return account from email', done => { 25 | const res = index.getEntityByEmail(obj.accounts[1].email); 26 | assert.strictEqual(typeof res, 'object'); 27 | assert.strictEqual(res.canonicalID, obj.accounts[1].canonicalID); 28 | done(); 29 | }); 30 | 31 | it('Should return account from key', done => { 32 | const res = index.getEntityByKey(obj.accounts[0].keys[0].access); 33 | assert.strictEqual(typeof res, 'object'); 34 | assert.strictEqual(res.arn, obj.accounts[0].arn); 35 | done(); 36 | }); 37 | 38 | it('Should return account from shortID', done => { 39 | const res = index.getEntityByShortId(obj.accounts[0].shortid); 40 | assert.strictEqual(typeof res, 'object'); 41 | assert.strictEqual(res.arn, obj.accounts[0].arn); 42 | done(); 43 | }); 44 | 45 | it('should index account without keys', done => { 46 | should._exec = () => { 47 | index = new Indexer(obj); 48 | const res = index.getEntityByEmail(obj.accounts[0].email); 49 | assert.strictEqual(typeof res, 'object'); 50 | assert.strictEqual(res.arn, obj.accounts[0].arn); 51 | done(); 52 | }; 53 | should.missingField(obj, 'accounts.0.keys'); 54 | }); 55 | 56 | it('should index account without users', done => { 57 | should._exec = () => { 58 | index = new Indexer(obj); 59 | const res = index.getEntityByEmail(obj.accounts[0].email); 60 | assert.strictEqual(typeof res, 'object'); 61 | assert.strictEqual(res.arn, obj.accounts[0].arn); 62 | done(); 63 | }; 64 | should.missingField(obj, 'accounts.0.users'); 65 | }); 66 | }); 67 | -------------------------------------------------------------------------------- /tests/unit/auth/in_memory/sample_authdata.json: -------------------------------------------------------------------------------- 1 | { 2 | "accounts": [{ 3 | "name": "Bart", 4 | "email": "sampleaccount1@sampling.com", 5 | "arn": "arn:aws:iam::123456789012:root", 6 | "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", 7 | "shortid": "123456789012", 8 | "keys": [{ 9 | "access": "accessKey1", 10 | "secret": "verySecretKey1" 11 | }] 12 | }, { 13 | "name": "Lisa", 14 | "email": "sampleaccount2@sampling.com", 15 | "arn": "arn:aws:iam::123456789013:root", 16 | "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf", 17 | "shortid": "123456789013", 18 | "keys": [{ 19 | "access": "accessKey2", 20 | "secret": "verySecretKey2" 21 | }] 22 | }] 23 | } 24 | -------------------------------------------------------------------------------- /tests/unit/auth/in_memory/sample_authdata_refresh.json: -------------------------------------------------------------------------------- 1 | { 2 | "accounts": [{ 3 | "name": "Zenko", 4 | "email": "sampleaccount4@sampling.com", 5 | "arn": "aws::iam:123456789015:root", 6 | "canonicalID": "newCanId", 7 | "shortid": "123456789015", 8 | "keys": [{ 9 | "access": "accessKeyZenko", 10 | "secret": "verySecretKeyZenko" 11 | }] 12 | }] 13 | } 14 | -------------------------------------------------------------------------------- /tests/unit/auth/v2/checkRequestExpiry.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict'; // eslint-disable-line strict 2 | 3 | const assert = require('assert'); 4 | 5 | const checkRequestExpiry = 6 | require('../../../../lib/auth/v2/checkRequestExpiry').default; 7 | const DummyRequestLogger = require('../../helpers').DummyRequestLogger; 8 | const errors = require('../../../../index').errors; 9 | 10 | const log = new DummyRequestLogger(); 11 | 12 | describe('checkTimestamp for timecheck in header auth', () => { 13 | it('should return AccessDenied error if the date in the ' + 14 | 'header is before epochTime', () => { 15 | const timestamp = new Date('1950-01-01'); 16 | const timeoutResult = checkRequestExpiry(timestamp, log); 17 | assert.deepStrictEqual(timeoutResult, errors.AccessDenied); 18 | }); 19 | 20 | it('should return RequestTimeTooSkewed error if the date in the ' + 21 | 'header is more than 15 minutes old', () => { 22 | const timestamp = new Date(Date.now() - 16 * 60000); 23 | const timeoutResult = checkRequestExpiry(timestamp, log); 24 | assert.deepStrictEqual(timeoutResult, errors.RequestTimeTooSkewed); 25 | }); 26 | 27 | it('should return RequestTimeTooSkewed error if the date in ' + 28 | 'the header is more than 15 minutes in the future', () => { 29 | const timestamp = new Date(Date.now() + 16 * 60000); 30 | const timeoutResult = checkRequestExpiry(timestamp, log); 31 | assert.deepStrictEqual(timeoutResult, errors.RequestTimeTooSkewed); 32 | }); 33 | 34 | it('should return no error if the date in the header is ' + 35 | 'within 15 minutes of current time', () => { 36 | const timestamp = new Date(); 37 | const timeoutResult = checkRequestExpiry(timestamp, log); 38 | assert.deepStrictEqual(timeoutResult, undefined); 39 | }); 40 | }); 41 | -------------------------------------------------------------------------------- /tests/unit/auth/v2/headerAuthCheck.spec.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | 3 | const headerAuthCheck = 4 | require('../../../../lib/auth/v2/headerAuthCheck').check; 5 | const DummyRequestLogger = require('../../helpers').DummyRequestLogger; 6 | 7 | const log = new DummyRequestLogger(); 8 | 9 | describe('v2: headerAuthCheck', () => { 10 | [ 11 | { token: undefined, error: false }, 12 | { token: 'invalid-token', error: true }, 13 | { token: 'a'.repeat(128), error: false }, 14 | ].forEach(test => it(`test with token(${test.token})`, () => { 15 | const request = { 16 | headers: { 17 | 'x-amz-security-token': test.token, 18 | }, 19 | }; 20 | const res = headerAuthCheck(request, log, {}); 21 | if (test.error) { 22 | assert.notStrictEqual(res.err, undefined); 23 | assert.strictEqual(res.err.is.InvalidToken, true); 24 | } else { 25 | assert.notStrictEqual(res.err, undefined); 26 | assert.notStrictEqual(res.err.is.InvalidToken, true); 27 | } 28 | })); 29 | }); 30 | -------------------------------------------------------------------------------- /tests/unit/auth/v2/publicAccess.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict'; // eslint-disable-line strict 2 | 3 | const assert = require('assert'); 4 | 5 | const errors = require('../../../../lib/errors').default; 6 | const auth = require('../../../../lib/auth/auth').server.doAuth; 7 | const AuthInfo = require('../../../../lib/auth/AuthInfo').default; 8 | const constants = require('../../../../lib/constants'); 9 | const DummyRequestLogger = require('../../helpers.js').DummyRequestLogger; 10 | const RequestContext = 11 | require('../../../../lib/policyEvaluator/RequestContext').default; 12 | 13 | const logger = new DummyRequestLogger(); 14 | 15 | describe('Public Access', () => { 16 | it('should grant access to a user that provides absolutely' + 17 | 'no authentication information and should assign that user the ' + 18 | 'All Users Group accessKey', done => { 19 | const request = { 20 | method: 'GET', 21 | headers: { host: 's3.amazonaws.com' }, 22 | url: '/bucket', 23 | query: {}, 24 | }; 25 | const singleRequstContext = new RequestContext(request.headers, 26 | request.query, request.bucketName, request.objectKey, 27 | undefined, undefined, 28 | 'bucketGet', 's3'); 29 | const requestContext = [singleRequstContext, singleRequstContext]; 30 | const publicAuthInfo = new AuthInfo({ 31 | canonicalID: constants.publicId, 32 | }); 33 | auth(request, logger, (err, authInfo) => { 34 | assert.strictEqual(err, null); 35 | assert.strictEqual(authInfo.getCanonicalID(), 36 | publicAuthInfo.getCanonicalID()); 37 | done(); 38 | }, 's3', requestContext); 39 | }); 40 | 41 | it('should not grant access to a request that contains ' + 42 | 'an authorization header without proper credentials', done => { 43 | const request = { 44 | method: 'GET', 45 | headers: { 46 | host: 's3.amazonaws.com', 47 | authorization: 'noAuth', 48 | }, 49 | url: '/bucket', 50 | query: {}, 51 | }; 52 | const requestContext = [new RequestContext(request.headers, 53 | request.query, request.bucketName, request.objectKey, 54 | undefined, undefined, 55 | 'bucketGet', 's3')]; 56 | auth(request, logger, err => { 57 | assert.deepStrictEqual(err, errors.AccessDenied); 58 | done(); 59 | }, 's3', requestContext); 60 | }); 61 | }); 62 | -------------------------------------------------------------------------------- /tests/unit/auth/v4/generateV4Headers.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict'; // eslint-disable-line strict 2 | 3 | const assert = require('assert'); 4 | const http = require('http'); 5 | 6 | const { generateV4Headers } = require('../../../../lib/auth/auth').client; 7 | 8 | const host = 'localhost:8000'; 9 | const token = 'token'; 10 | const data = 'data'; 11 | 12 | describe('v4 header generation', () => { 13 | it('should add x-amz-security-token if needed', done => { 14 | const req = new http.OutgoingMessage(); 15 | req.setHeader('host', host); 16 | 17 | generateV4Headers(req, data, 'accessKey', 'secretKey', 'iam', null, token); 18 | 19 | try { 20 | assert.deepStrictEqual(req.getHeader('x-amz-security-token'), token); 21 | return done(); 22 | } catch (err) { 23 | return done(err); 24 | } 25 | }); 26 | 27 | it('should not add x-amz-security-token by default', done => { 28 | const req = new http.OutgoingMessage(); 29 | req.setHeader('host', host); 30 | 31 | generateV4Headers(req, data, 'accessKey', 'secretKey', 'iam'); 32 | 33 | try { 34 | assert.deepStrictEqual(req.getHeader('x-amz-security-token'), undefined); 35 | return done(); 36 | } catch (err) { 37 | return done(err); 38 | } 39 | }); 40 | }); 41 | -------------------------------------------------------------------------------- /tests/unit/auth/v4/signingKey.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict'; // eslint-disable-line strict 2 | 3 | const assert = require('assert'); 4 | 5 | const calculateSigningKey = 6 | require('../../../../lib/auth/backends/in_memory/vaultUtilities') 7 | .calculateSigningKey; 8 | 9 | describe('v4 signing key calculation', () => { 10 | it('should calculate a signing key in accordance with AWS rules', () => { 11 | const secretKey = 'verySecretKey1'; 12 | const region = 'us-east-1'; 13 | const scopeDate = '20160209'; 14 | const expectedOutput = '5c19fe2935aa4f967549048b6daa85635fb47' + 15 | 'be2938b0899177e5906d4b17221'; 16 | const actualOutput = calculateSigningKey(secretKey, region, scopeDate); 17 | const buff = Buffer.from(actualOutput, 'binary').toString('hex'); 18 | assert.strictEqual(buff, expectedOutput); 19 | }); 20 | }); 21 | -------------------------------------------------------------------------------- /tests/unit/ipCheck.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict'; // eslint-disable-line strict 2 | 3 | const assert = require('assert'); 4 | const ipCheck = require('../../lib/ipCheck'); 5 | const ipaddr = require('ipaddr.js'); 6 | 7 | function parseValidIpCheck(ip, sup) { 8 | const actualRes = ipCheck.parseIp(ip); 9 | assert(actualRes instanceof sup); 10 | } 11 | 12 | function parseInvalidIpCheck(ip) { 13 | const actualRes = ipCheck.parseIp(ip); 14 | assert.deepStrictEqual(actualRes, {}); 15 | } 16 | 17 | function cidrMatchCheck(cidr, ip, expectedRes) { 18 | const parsed = ipCheck.parseIp(ip); 19 | const actualRes = ipCheck.checkIPinRangeOrMatch(cidr, parsed); 20 | assert.strictEqual(actualRes, expectedRes); 21 | } 22 | 23 | function cidrListMatchCheck(cidrList, ip, expectedRes) { 24 | const actualRes = ipCheck.ipMatchCidrList(cidrList, ip); 25 | assert.strictEqual(actualRes, expectedRes); 26 | } 27 | 28 | describe('Parse IP address', () => { 29 | it('should parse IPv4 address', 30 | () => parseValidIpCheck('192.168.1.1', ipaddr.IPv4)); 31 | 32 | it('should parse IPv6 address', 33 | () => parseValidIpCheck('2001:cdba::3257:9652', ipaddr.IPv6)); 34 | 35 | it('should parse IPv4 mapped IPv6 address', 36 | // ::ffff:c0a8:101 mapped for 192.168.1.1 37 | () => parseValidIpCheck('::ffff:c0a8:101', ipaddr.IPv4)); 38 | 39 | ['260.384.2.1', 'INVALID', '', null, undefined].forEach(item => { 40 | it(`should return empty object for invalid IP address: (${item})`, 41 | () => parseInvalidIpCheck(item)); 42 | }); 43 | }); 44 | 45 | describe('Check IP matches CIDR range', () => { 46 | it('should match IP in a range', 47 | () => cidrMatchCheck('192.168.1.0/24', '192.168.1.1', true)); 48 | 49 | it('should not match IP not in a range', 50 | () => cidrMatchCheck('192.168.1.0/24', '127.0.0.1', false)); 51 | 52 | it('should match if range equals IP', 53 | () => cidrMatchCheck('192.168.1.1', '192.168.1.1', true)); 54 | 55 | 56 | ['260.384.2.1', 'INVALID', '', null, undefined].forEach(item => { 57 | it(`should not match for invalid IP: (${item})`, 58 | () => cidrMatchCheck('192.168.1.0/24', item, false)); 59 | }); 60 | }); 61 | 62 | describe('Check IP matches a list of CIDR ranges', () => { 63 | it('should match IP in a valid range', 64 | () => cidrListMatchCheck(['192.168.1.0/24', '192.168.100.14/24', 65 | '2001:db8::'], '192.168.100.1', true)); 66 | 67 | [ 68 | [['127.0.0.1'], '127.0.0.2'], 69 | [['192.168.1.1'], '192.168.1.1'], 70 | ].forEach(item => 71 | it(`should match IP ${item[0][0]} without CIDR range`, 72 | () => cidrListMatchCheck(item[0], item[1], true)), 73 | ); 74 | 75 | it('should not range match if CIDR range is not provided', 76 | () => cidrListMatchCheck(['192.168.1.1'], '192.168.1.3', false)); 77 | }); 78 | -------------------------------------------------------------------------------- /tests/unit/jsutil.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict';// eslint-disable-line 2 | 3 | const assert = require('assert'); 4 | const jsutil = require('../../index').jsutil; 5 | 6 | describe('JSUtil', () => { 7 | describe('once', () => { 8 | it('should call the wrapped function only once when invoked ' + 9 | 'multiple times', 10 | done => { 11 | let value = 42; 12 | let value2 = 51; 13 | 14 | const wrapOnce = jsutil.once(expectArg => { 15 | assert.strictEqual(expectArg, 'foo'); 16 | value += 1; 17 | return value; 18 | }); 19 | const wrapOnce2 = jsutil.once(expectArg => { 20 | assert.strictEqual(expectArg, 'foo2'); 21 | value2 += 1; 22 | return value2; 23 | }); 24 | assert.strictEqual(wrapOnce('foo'), 43); 25 | assert.strictEqual(wrapOnce2('foo2'), 52); 26 | assert.strictEqual(wrapOnce('bar'), 43); 27 | assert.strictEqual(wrapOnce2('bar2'), 52); 28 | done(); 29 | }); 30 | }); 31 | }); 32 | -------------------------------------------------------------------------------- /tests/unit/models/ObjectMDAmzRestore.spec.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const ObjectMDAmzRestore = require('../../../lib/models/ObjectMDAmzRestore').default; 3 | 4 | const amzRestore = new ObjectMDAmzRestore(false, new Date()); 5 | 6 | describe('ObjectMDAmzRestore value', () => { 7 | it('should return the correct value', () => { 8 | const amzRestoreObj = amzRestore.getValue(); 9 | assert.deepStrictEqual(amzRestoreObj, amzRestore); 10 | }); 11 | }); 12 | 13 | describe('ObjectMDAmzRestore setters/getters', () => { 14 | it('should control the ongoing-request attribute', () => { 15 | const ongoing = true; 16 | const wrongOngoingRequest = 'bad'; 17 | amzRestore.setOngoingRequest(ongoing); 18 | assert.deepStrictEqual(amzRestore.getOngoingRequest(), 19 | ongoing); 20 | assert.throws(() => { 21 | amzRestore.setOngoingRequest(); 22 | }); 23 | assert.throws(() => { 24 | amzRestore.setOngoingRequest(wrongOngoingRequest); 25 | }); 26 | }); 27 | it('should control the expiry-date attribute', () => { 28 | const expiry = new Date(100); 29 | const wrongExpiryDate = 'bad'; 30 | amzRestore.setExpiryDate(expiry); 31 | assert.deepStrictEqual(amzRestore.getExpiryDate(), 32 | expiry); 33 | assert.throws(() => { 34 | amzRestore.setExpiryDate(wrongExpiryDate); 35 | }); 36 | }); 37 | }); 38 | -------------------------------------------------------------------------------- /tests/unit/network/probe/ProbeServer.spec.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const http = require('http'); 3 | const index = require('../../../../'); 4 | const { ProbeServer } = index.network.probe.ProbeServer; 5 | 6 | function makeRequest(method, uri, cb) { 7 | const params = { 8 | hostname: 'localhost', 9 | port: 4042, 10 | path: uri, 11 | method, 12 | }; 13 | const req = http.request(params); 14 | req.setNoDelay(true); 15 | req.on('response', res => { 16 | cb(undefined, res); 17 | }); 18 | req.on('error', err => { 19 | assert.ifError(err); 20 | cb(err); 21 | }).end(); 22 | } 23 | 24 | describe('network.probe.ProbeServer', () => { 25 | /** @type {ProbeServer} */ 26 | let server; 27 | 28 | beforeEach(done => { 29 | server = new ProbeServer({ port: 4042 }); 30 | server._cbOnListening = done; 31 | server.start(); 32 | }); 33 | 34 | afterEach(done => { 35 | server.stop(); 36 | done(); 37 | }); 38 | 39 | it('error on bad method', done => { 40 | makeRequest('POST', '/unused', (err, res) => { 41 | assert.ifError(err); 42 | assert.strictEqual(res.statusCode, 405); 43 | done(); 44 | }); 45 | }); 46 | 47 | it('error on invalid route', done => { 48 | makeRequest('GET', '/unused', (err, res) => { 49 | assert.ifError(err); 50 | assert.strictEqual(res.statusCode, 400); 51 | done(); 52 | }); 53 | }); 54 | 55 | it('allows probe to handle requests', done => { 56 | server.addHandler('/check', res => { 57 | res.writeHead(200); 58 | res.end(); 59 | }); 60 | makeRequest('GET', '/check', (err, res) => { 61 | assert.ifError(err); 62 | assert.strictEqual(res.statusCode, 200); 63 | done(); 64 | }); 65 | }); 66 | 67 | it('accepts array of paths', done => { 68 | server.addHandler(['/check', '/probe'], res => { 69 | res.writeHead(200); 70 | res.end(); 71 | }); 72 | let calls = 0; 73 | makeRequest('GET', '/check', (err, res) => { 74 | assert.ifError(err); 75 | assert.strictEqual(res.statusCode, 200); 76 | calls++; 77 | if (calls === 2) { 78 | done(); 79 | } 80 | }); 81 | makeRequest('GET', '/probe', (err, res) => { 82 | assert.ifError(err); 83 | assert.strictEqual(res.statusCode, 200); 84 | calls++; 85 | if (calls === 2) { 86 | done(); 87 | } 88 | }); 89 | }); 90 | }); 91 | -------------------------------------------------------------------------------- /tests/unit/network/probe/Utils.spec.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const errors = require('../../../../lib/errors').default; 3 | const errorInstances = require('../../../../lib/errors').default; 4 | const { sendError, sendSuccess } = require('../../../../lib/network/probe/Utils'); 5 | const sinon = require('sinon'); 6 | 7 | describe('network.probe.Utils', () => { 8 | let mockLogger; 9 | 10 | beforeEach(() => { 11 | mockLogger = { 12 | debug: sinon.fake(), 13 | }; 14 | }); 15 | 16 | afterEach(() => { 17 | sinon.restore(); 18 | }); 19 | 20 | it('send success will return 200 OK', done => { 21 | const mockRes = { 22 | writeHead: sinon.fake(status => assert.strictEqual(200, status)), 23 | end: sinon.fake(msg => { 24 | assert.strictEqual(msg, 'OK'); 25 | done(); 26 | }), 27 | }; 28 | sendSuccess(mockRes, mockLogger); 29 | }); 30 | 31 | it('send success will return 200 and optional message', done => { 32 | const mockRes = { 33 | writeHead: sinon.fake(status => assert.strictEqual(200, status)), 34 | end: sinon.fake(msg => { 35 | assert.strictEqual(msg, 'Granted'); 36 | done(); 37 | }), 38 | }; 39 | sendSuccess(mockRes, mockLogger, 'Granted'); 40 | }); 41 | 42 | it('send error will return send an Arsenal Error and code', done => { 43 | const mockRes = { 44 | writeHead: sinon.fake(status => assert.strictEqual(405, status)), 45 | end: sinon.fake(msg => { 46 | assert.deepStrictEqual( 47 | JSON.parse(msg), 48 | { 49 | errorType: 'MethodNotAllowed', 50 | errorMessage: errorInstances.MethodNotAllowed.description, 51 | }, 52 | ); 53 | done(); 54 | }), 55 | }; 56 | sendError(mockRes, mockLogger, errors.MethodNotAllowed); 57 | }); 58 | 59 | it('send error will return send an Arsenal Error and code using optional message', done => { 60 | const mockRes = { 61 | writeHead: sinon.fake(status => assert.strictEqual(405, status)), 62 | end: sinon.fake(msg => { 63 | assert.deepStrictEqual( 64 | JSON.parse(msg), 65 | { 66 | errorType: 'MethodNotAllowed', 67 | errorMessage: 'Very much not allowed', 68 | }, 69 | ); 70 | done(); 71 | }), 72 | }; 73 | sendError(mockRes, mockLogger, errors.MethodNotAllowed, 'Very much not allowed'); 74 | }); 75 | }); 76 | -------------------------------------------------------------------------------- /tests/unit/network/rest/utils.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict'; // eslint-disable-line 2 | 3 | const assert = require('assert'); 4 | const constants = require('../../../../lib/constants'); 5 | const { parseURL } = require('../../../../lib/network/rest/utils'); 6 | 7 | describe('parseURL function', () => { 8 | [ 9 | { 10 | inputUrl: `${constants.passthroughFileURL}/test`, 11 | expectedKey: 'test', 12 | }, 13 | { 14 | inputUrl: `${constants.passthroughFileURL}/test with spaces`, 15 | expectedKey: 'test with spaces', 16 | }, 17 | { 18 | inputUrl: `${constants.passthroughFileURL}` + 19 | '/test%20with%20encoded%20spaces', 20 | expectedKey: 'test with encoded spaces', 21 | }, 22 | ].forEach(testCase => { 23 | const { inputUrl, expectedKey } = testCase; 24 | 25 | it(`should return ${expectedKey} with url "${inputUrl}"`, 26 | () => { 27 | const pathInfo = parseURL(inputUrl, true); 28 | assert.strictEqual(pathInfo.key, expectedKey); 29 | }); 30 | }); 31 | }); 32 | -------------------------------------------------------------------------------- /tests/unit/patches/creds.json: -------------------------------------------------------------------------------- 1 | { 2 | "privateKey": "-----BEGIN RSA PRIVATE KEY-----\r\nMIIEowIBAAKCAQEAj13sSYE40lAX2qpBvfdGfcSVNtBf8i5FH+E8FAhORwwPu+2S\r\n3yBQbgwHq30WWxunGb1NmZL1wkVZ+vf12DtxqFRnMA08LfO4oO6oC4V8XfKeuHyJ\r\n1qlaKRINz6r9yDkTHtwWoBnlAINurlcNKgGD5p7D+G26Chbr/Oo0ZwHula9DxXy6\r\neH8/bJ5/BynyNyyWRPoAO+UkUdY5utkFCUq2dbBIhovMgjjikf5p2oWqnRKXc+JK\r\nBegr6lSHkkhyqNhTmd8+wA+8Cace4sy1ajY1t5V4wfRZea5vwl/HlyyKodvHdxng\r\nJgg6H61JMYPkplY6Gr9OryBKEAgq02zYoYTDfwIDAQABAoIBAAuDYGlavkRteCzw\r\nRU1LIVcSRWVcgIgDXTu9K8T0Ec0008Kkxomyn6LmxmroJbZ1VwsDH8s4eRH73ckA\r\nxrZxt6Pr+0lplq6eBvKtl8MtGhq1VDe+kJczjHEF6SQHOFAu/TEaPZrn2XMcGvRX\r\nO1BnRL9tepFlxm3u/06VRFYNWqqchM+tFyzLu2AuiuKd5+slSX7KZvVgdkY1ErKH\r\ngB75lPyhPb77C/6ptqUisVMSO4JhLhsD0+ekDVY982Sb7KkI+szdWSbtMx9Ek2Wo\r\ntXwJz7I8T7IbODy9aW9G+ydyhMDFmaEYIaDVFKJj5+fluNza3oQ5PtFNVE50GQJA\r\nsisGqfECgYEAwpkwt0KpSamSEH6qknNYPOwxgEuXWoFVzibko7is2tFPvY+YJowb\r\n68MqHIYhf7gHLq2dc5Jg1TTbGqLECjVxp4xLU4c95KBy1J9CPAcuH4xQLDXmeLzP\r\nJ2YgznRocbzAMCDAwafCr3uY9FM7oGDHAi5bE5W11xWx+9MlFExL3JkCgYEAvJp5\r\nf+JGN1W037bQe2QLYUWGszewZsvplnNOeytGQa57w4YdF42lPhMz6Kc/zdzKZpN9\r\njrshiIDhAD5NCno6dwqafBAW9WZl0sn7EnlLhD4Lwm8E9bRHnC9H82yFuqmNrzww\r\nzxBCQogJISwHiVz4EkU48B283ecBn0wT/fAa19cCgYEApKWsnEHgrhy1IxOpCoRh\r\nUhqdv2k1xDPN/8DUjtnAFtwmVcLa/zJopU/Zn4y1ZzSzjwECSTi+iWZRQ/YXXHPf\r\nl92SFjhFW92Niuy8w8FnevXjF6T7PYiy1SkJ9OR1QlZrXc04iiGBDazLu115A7ce\r\nanACS03OLw+CKgl6Q/RR83ECgYBCUngDVoimkMcIHHt3yJiP3ikeAKlRnMdJlsa0\r\nXWVZV4hCG3lDfRXsnEgWuimftNKf+6GdfYSvQdLdiQsCcjT5A4uLsQTByv5nf4uA\r\n1ZKOsFrmRrARzxGXhLDikvj7yP//7USkq+0BBGFhfuAvl7fMhPceyPZPehqB7/jf\r\nxX1LBQKBgAn5GgSXzzS0e06ZlP/VrKxreOHa5Z8wOmqqYQ0QTeczAbNNmuITdwwB\r\nNkbRqpVXRIfuj0BQBegAiix8om1W4it0cwz54IXBwQULxJR1StWxj3jo4QtpMQ+z\r\npVPdB1Ilb9zPV1YvDwRfdS1xsobzznAx56ecsXduZjs9mF61db8Q\r\n-----END RSA PRIVATE KEY-----\r\n", 3 | "publicKey": "-----BEGIN PUBLIC KEY-----\r\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAj13sSYE40lAX2qpBvfdG\r\nfcSVNtBf8i5FH+E8FAhORwwPu+2S3yBQbgwHq30WWxunGb1NmZL1wkVZ+vf12Dtx\r\nqFRnMA08LfO4oO6oC4V8XfKeuHyJ1qlaKRINz6r9yDkTHtwWoBnlAINurlcNKgGD\r\n5p7D+G26Chbr/Oo0ZwHula9DxXy6eH8/bJ5/BynyNyyWRPoAO+UkUdY5utkFCUq2\r\ndbBIhovMgjjikf5p2oWqnRKXc+JKBegr6lSHkkhyqNhTmd8+wA+8Cace4sy1ajY1\r\nt5V4wfRZea5vwl/HlyyKodvHdxngJgg6H61JMYPkplY6Gr9OryBKEAgq02zYoYTD\r\nfwIDAQAB\r\n-----END PUBLIC KEY-----\r\n", 4 | "accessKey": "QXP3VDG3SALNBX2QBJ1C", 5 | "secretKey": "K5FyqZo5uFKfw9QBtn95o6vuPuD0zH/1seIrqPKqGnz8AxALNSx6EeRq7G1I6JJpS1XN13EhnwGn2ipsml3Uf2fQ00YgEmImG8wzGVZm8fWotpVO4ilN4JGyQCah81rNX4wZ9xHqDD7qYR5MyIERxR/osoXfctOwY7GGUjRKJfLOguNUlpaovejg6mZfTvYAiDF+PTO1sKUYqHt1IfKQtsK3dov1EFMBB5pWM7sVfncq/CthKN5M+VHx9Y87qdoP3+7AW+RCBbSDOfQgxvqtS7PIAf10mDl8k2kEURLz+RqChu4O4S0UzbEmtja7wa7WYhYKv/tM/QeW7kyNJMmnPg==", 6 | "decryptedSecretKey": "n7PSZ3U6SgerF9PCNhXYsq3S3fRKVGdZTicGV8Ur", 7 | "canonicalId": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", 8 | "userName": "arsenal-0" 9 | } 10 | -------------------------------------------------------------------------------- /tests/unit/s3middleware/MD5Sum.spec.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const DummyRequest = require('../../utils/DummyRequest'); 3 | const MD5Sum = require('../../../lib/s3middleware/MD5Sum').default; 4 | const constants = require('../../../lib/constants'); 5 | 6 | function consume(stream) { 7 | stream.on('data', () => { }); 8 | } 9 | 10 | function testMD5(payload, expectedMD5, done) { 11 | const dummy = new DummyRequest({}, payload); 12 | const md5summer = new MD5Sum(); 13 | md5summer.on('hashed', () => { 14 | assert.strictEqual(md5summer.completedHash, expectedMD5); 15 | done(); 16 | }); 17 | dummy.pipe(md5summer); 18 | consume(md5summer); 19 | } 20 | 21 | describe('utilities.MD5Sum', () => { 22 | it('should work on empty request', done => { 23 | testMD5('', constants.emptyFileMd5, done); 24 | }); 25 | 26 | it('should work on SAY GRRRR!!! request', done => { 27 | testMD5('SAY GRRRR!!!', '986eb4a201192e8b1723a42c1468fb4e', done); 28 | }); 29 | 30 | it('should work on multiple MiB data stream', done => { 31 | /* 32 | * relies on a zero filled buffer and 33 | * split content in order to get multiple calls of _transform() 34 | * md5sum computed by hand with: 35 | * $ dd if=/dev/zero of=/dev/stdout bs=1M count=16 2>/dev/null | md5sum 36 | */ 37 | const buffer = Buffer.alloc(4 * 1024 * 1024); 38 | testMD5([buffer, buffer, buffer, buffer], 39 | '2c7ab85a893283e98c931e9511add182', done); 40 | }); 41 | }); 42 | -------------------------------------------------------------------------------- /tests/unit/s3middleware/azureHelpers/SubStreamingInterface.spec.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const stream = require('stream'); 3 | const SubStreamInterface = 4 | require('../../../../lib/s3middleware/azureHelpers/SubStreamInterface').default; 5 | 6 | describe('s3middleware SubStreamInterface.stopStreaming()', () => { 7 | const eventsEmitted = { 8 | sourceStreamUnpiped: false, 9 | currentStreamStopStreamingToAzure: false, 10 | currentStreamEnded: false, 11 | }; 12 | const expectedSequence = { 13 | sourceStreamUnpiped: 0, 14 | currentStreamStopStreamingToAzure: 1, 15 | currentStreamEnded: 2, 16 | }; 17 | const data = Buffer.alloc(100); 18 | let dataMarker = 0; 19 | let eventSequence = 0; 20 | const mockRequest = new stream.Readable({ 21 | read: () => { 22 | if (dataMarker >= data.length) { 23 | return mockRequest.push(null); 24 | } 25 | mockRequest.push(data.slice(dataMarker, dataMarker + 1)); 26 | dataMarker += 1; 27 | return undefined; 28 | }, 29 | }); 30 | const sourceStream = new stream.PassThrough(); 31 | const subStreamInterface = new SubStreamInterface(sourceStream); 32 | sourceStream.on('unpipe', () => { 33 | eventsEmitted.sourceStreamUnpiped = eventSequence++; 34 | }); 35 | subStreamInterface._currentStream.on('stopStreamingToAzure', () => { 36 | eventsEmitted.currentStreamStopStreamingToAzure = eventSequence++; 37 | }); 38 | subStreamInterface._currentStream.on('finish', () => { 39 | eventsEmitted.currentStreamEnded = eventSequence++; 40 | }); 41 | it('should stop streaming data and end current stream', done => { 42 | sourceStream.on('data', chunk => { 43 | const currentLength = subStreamInterface.getLengthCounter(); 44 | if (currentLength === 10) { 45 | Object.keys(eventsEmitted).forEach(key => { 46 | assert.strictEqual(eventsEmitted[key], false); 47 | }); 48 | assert.strictEqual(mockRequest._readableState.pipesCount, 1); 49 | return subStreamInterface.stopStreaming(mockRequest); 50 | } 51 | return subStreamInterface.write(chunk); 52 | }); 53 | mockRequest.pipe(sourceStream); 54 | setTimeout(() => { 55 | Object.keys(eventsEmitted).forEach(key => { 56 | assert.strictEqual(eventsEmitted[key], expectedSequence[key]); 57 | }); 58 | assert.strictEqual(subStreamInterface.getLengthCounter(), 10); 59 | assert.strictEqual(mockRequest._readableState.pipesCount, 0); 60 | return done(); 61 | }, 1000); 62 | }); 63 | }); 64 | -------------------------------------------------------------------------------- /tests/unit/s3middleware/azureHelpers/mpuUtils.spec.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | 3 | const azureMpuUtils = 4 | require('../../../../lib/s3middleware/azureHelpers/mpuUtils'); 5 | const padString = azureMpuUtils.padString; 6 | const getSubPartInfo = azureMpuUtils.getSubPartInfo; 7 | 8 | const padStringTests = [ 9 | { 10 | category: 'partNumber', 11 | strings: [1, 10, 100, 10000], 12 | expectedResults: ['00001', '00010', '00100', '10000'], 13 | }, { 14 | category: 'subPart', 15 | strings: [1, 50], 16 | expectedResults: ['01', '50'], 17 | }, { 18 | category: 'part', 19 | strings: ['test|'], 20 | expectedResults: 21 | ['test|%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%'], 22 | }, 23 | ]; 24 | 25 | const oneMb = 1024 * 1024; 26 | const oneHundredMb = oneMb * 100; 27 | const subPartInfoTests = [ 28 | { 29 | desc: '100 mb', 30 | size: oneHundredMb, 31 | expectedNumberSubParts: 1, 32 | expectedLastPartSize: oneHundredMb, 33 | }, { 34 | desc: '101 mb', 35 | size: oneHundredMb + oneMb, 36 | expectedNumberSubParts: 2, 37 | expectedLastPartSize: oneMb, 38 | }, { 39 | desc: '599 mb', 40 | size: 6 * oneHundredMb - oneMb, 41 | expectedNumberSubParts: 6, 42 | expectedLastPartSize: oneHundredMb - oneMb, 43 | }, { 44 | desc: '600 mb', 45 | size: 6 * oneHundredMb, 46 | expectedNumberSubParts: 6, 47 | expectedLastPartSize: oneHundredMb, 48 | }, 49 | ]; 50 | 51 | describe('s3middleware Azure MPU helper utility function', () => { 52 | padStringTests.forEach(test => { 53 | it(`padString should pad a ${test.category}`, done => { 54 | const result = test.strings.map(str => 55 | padString(str, test.category)); 56 | assert.deepStrictEqual(result, test.expectedResults); 57 | done(); 58 | }); 59 | }); 60 | 61 | subPartInfoTests.forEach(test => { 62 | const { desc, size, expectedNumberSubParts, expectedLastPartSize } 63 | = test; 64 | it('getSubPartInfo should return correct result for ' + 65 | `dataContentLength of ${desc}`, done => { 66 | const result = getSubPartInfo(size); 67 | const expectedLastPartIndex = expectedNumberSubParts - 1; 68 | assert.strictEqual(result.lastPartIndex, expectedLastPartIndex); 69 | assert.strictEqual(result.lastPartSize, expectedLastPartSize); 70 | done(); 71 | }); 72 | }); 73 | }); 74 | -------------------------------------------------------------------------------- /tests/unit/s3middleware/nullStream.spec.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const NullStream = require('../../../lib/s3middleware/nullStream').default; 3 | const MD5Sum = require('../../../lib/s3middleware/MD5Sum').default; 4 | 5 | const nullChunks = [ 6 | { size: 1, md5sum: '93b885adfe0da089cdf634904fd59f71' }, 7 | { size: 15000000, md5sum: 'e05ffc4ac83a1a4f4d545a3a4852383f' }, 8 | ]; 9 | 10 | function testNullChunk(size, range, expectedMD5, done) { 11 | const nullChunk = new NullStream(size, range); 12 | const digestStream = new MD5Sum(); 13 | digestStream.on('hashed', () => { 14 | assert.strictEqual(digestStream.completedHash, expectedMD5); 15 | done(); 16 | }); 17 | nullChunk.pipe(digestStream); 18 | digestStream.on('data', () => {}); 19 | } 20 | 21 | describe('s3middleware.NullStream', () => { 22 | for (let i = 0; i < nullChunks.length; ++i) { 23 | const size = nullChunks[i].size; 24 | const md5sum = nullChunks[i].md5sum; 25 | it(`should generate ${size} null bytes by size`, done => { 26 | testNullChunk(size, null, md5sum, done); 27 | }); 28 | it(`should generate ${size} null bytes by range`, done => { 29 | const dummyOffset = 9320954; 30 | testNullChunk(0, [dummyOffset, dummyOffset + size - 1], 31 | md5sum, done); 32 | }); 33 | } 34 | }); 35 | -------------------------------------------------------------------------------- /tests/unit/s3middleware/prepareStream.spec.js: -------------------------------------------------------------------------------- 1 | const sinon = require('sinon'); 2 | const { default: Vault } = require('../../../lib/auth/Vault'); 3 | const { prepareStream } = require('../../../lib/s3middleware/prepareStream'); 4 | const { Transform } = require('stream'); 5 | const DummyRequestLogger = require('../helpers').DummyRequestLogger; 6 | 7 | const log = new DummyRequestLogger(); 8 | 9 | describe('prepareStream', () => { 10 | const vault = new Vault(); 11 | const stream = { 12 | headers: {}, 13 | pipe: sinon.stub(), 14 | }; 15 | 16 | beforeEach(() => { 17 | stream.headers['x-amz-content-sha256'] = 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'; 18 | }); 19 | 20 | it('should return the stream if sha256 header is not streaming', () => { 21 | stream.headers['x-amz-content-sha256'] = 'NOT-STREAMING'; 22 | const result = prepareStream(stream, {}, vault, log); 23 | expect(result).toEqual(stream); 24 | }); 25 | 26 | it('should pipe using V4Transform if sha256 header is streaming', () => { 27 | const result = prepareStream(stream, {}, vault, log); 28 | expect(result).toBeInstanceOf(Transform); 29 | }); 30 | }); 31 | 32 | -------------------------------------------------------------------------------- /tests/unit/s3middleware/processMpuParts.spec.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const crypto = require('crypto'); 3 | 4 | const { createAggregateETag } = 5 | require('../../../lib/s3middleware/processMpuParts'); 6 | 7 | describe('createAggregateETag', () => { 8 | [{ 9 | partETags: ['3858f62230ac3c915f300c664312c63f'], 10 | aggregateETag: 'c4529dc85643bb0c5a96e46587377777-1', 11 | }, { 12 | partETags: ['ffc88b4ca90a355f8ddba6b2c3b2af5c', 13 | 'd067a0fa9dc61a6e7195ca99696b5a89'], 14 | aggregateETag: '620e8b191a353bdc9189840bb3904928-2', 15 | }, { 16 | partETags: ['ffc88b4ca90a355f8ddba6b2c3b2af5c', 17 | 'd067a0fa9dc61a6e7195ca99696b5a89', 18 | '49dcd91231f801159e893fb5c6674985', 19 | '1292a1f4afecfeb84e1b200389d1c904', 20 | '6b70b0751c98492074a7359f0f70d76d', 21 | '5c55c71b3b582f6b700f83bb834f2430', 22 | '84562b55618378a7ac5cfcbc7f3b2ceb', 23 | 'b5693c44bad7a2cf51c82c6a2fe1a4b6', 24 | '628b37ac2dee9c123cd2e3e2e486eb27', 25 | '4cacc7e3b7933e54422243964db169af', 26 | '0add1fb9122cc9df84aee7c4bb86d658', 27 | '5887704d69ee209f32c9314c345c8084', 28 | '374e87eeee83bed471b78eefc8d7e28e', 29 | '4e2af9f5fa8b64b19f78ddfbcfcab148', 30 | '8e06231275f3afe7953fc7d57b65723f', 31 | 'c972158cb957cf48e18b475b908d5d82', 32 | '311c2324dd756c9655129de049f69c9b', 33 | '0188a9df3e1c4ce18f81e4ba24c672a0', 34 | '1a15c4da6038a6626ad16473712eb358', 35 | 'd13c52938d8e0f01192d16b0de17ea4c'], 36 | aggregateETag: 'd3d5a0ab698dd360e755a467f7899e7e-20', 37 | }].forEach(test => { 38 | it(`should compute aggregate ETag with ${test.partETags.length} parts`, 39 | () => { 40 | const aggregateETag = createAggregateETag(test.partETags); 41 | assert.strictEqual(aggregateETag, test.aggregateETag); 42 | }); 43 | }); 44 | 45 | it('should compute aggregate ETag with 10000 parts', () => { 46 | const partETags = []; 47 | for (let i = 0; i < 10000; ++i) { 48 | const md5hash = crypto.createHash('md5'); 49 | md5hash.update(`part${i}`, 'binary'); 50 | partETags.push(md5hash.digest('hex')); 51 | } 52 | const aggregateETag = createAggregateETag(partETags); 53 | assert.strictEqual( 54 | aggregateETag, 'bff290751e485f06dcc0203c77ed2fd9-10000'); 55 | }); 56 | }); 57 | -------------------------------------------------------------------------------- /tests/unit/s3middleware/userMetadata.spec.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const { getMetaHeaders } = require('../../../lib/s3middleware/userMetadata'); 3 | const { maximumMetaHeadersSize } = require('../../../lib/constants'); 4 | 5 | function genMaxSizeMetaHeaders() { 6 | const metaHeaders = {}; 7 | const counter = 8; 8 | const bytesPerHeader = 9 | (maximumMetaHeadersSize / counter); 10 | for (let i = 0; i < counter; i++) { 11 | const key = `x-amz-meta-header${i}`; 12 | const valueLength = bytesPerHeader - key.length; 13 | metaHeaders[key] = '0'.repeat(valueLength); 14 | } 15 | return metaHeaders; 16 | } 17 | 18 | describe('get meta headers', () => { 19 | it('returns no error for metadata length equal to maximum allowed', () => { 20 | const headers = genMaxSizeMetaHeaders(); 21 | assert.doesNotThrow(() => { 22 | getMetaHeaders(headers); 23 | }); 24 | }); 25 | it('returns error for metadata length equal to maximum allowed plus one', () => { 26 | const headers = genMaxSizeMetaHeaders(); 27 | headers['x-amz-meta-header0'] = `${headers['x-amz-meta-header0']}1`; 28 | const result = getMetaHeaders(headers); 29 | assert(result instanceof Error); 30 | assert.strictEqual(result.is.MetadataTooLarge, true); 31 | }); 32 | }); 33 | -------------------------------------------------------------------------------- /tests/unit/s3routes/routesUtils/isValidObjectKey.spec.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const routesUtils = require('../../../../lib/s3routes/routesUtils'); 3 | 4 | const bannedStr = 'banned'; 5 | const prefixBlacklist = []; 6 | 7 | // byte size of 915 8 | const keyutf8 = '%EA%9D%8B崰㈌㒈保轖䳷䀰⺩ቆ楪秲ⴝ㿅鼎ꓜ퇬枅࿷염곞召㸾⌙ꪊᆐ庍뉆䌗幐鸆䛃➟녩' + 9 | 'ˍ뙪臅⠙≼绒벊냂詴 끴鹲萯⇂㭢䈊퉉楝舳㷖족痴䧫㾵᏷ำꎆ꼵껪멷㄀誕㳓腜쒃컹㑻鳃삚舿췈孨੦⮀NJ곓⵪꺼꜈' + 10 | '嗼뫘悕錸瑺⁤륒㜓垻ㆩꝿ詀펉ᆙ舑䜾힑藪碙ꀎꂰ췊Ᏻ 㘺幽醛잯ද汧Ꟑꛒⶨ쪸숞헹㭔ꡔᘼ뺓ᡆ᡾ᑟ䅅퀭耓弧⢠⇙' + 11 | '폪ް蛧⃪Ἔ돫ꕢ븥ヲ캂䝄쟐颺ᓾ둾Ұ껗礞ᾰ瘹蒯硳풛瞋襎奺熝妒컚쉴⿂㽝㝳駵鈚䄖戭䌸᫲ᇁ䙪鸮ᐴ稫ⶭ뀟ھ⦿' + 12 | '䴳稉ꉕ捈袿놾띐✯伤䃫⸧ꠏ瘌틳藔ˋ㫣敀䔩㭘식↴⧵佶痊牌ꪌ搒꾛æᤈべ쉴挜敗羥誜嘳ֶꫜ걵ࣀ묟ኋ拃秷膤䨸菥' + 13 | '䟆곘縧멀煣卲챸⧃⏶혣ਔ뙞밺㊑ک씌촃Ȅ頰ᖅ懚ホῐ꠷㯢먈㝹୥밷㮇䘖桲阥黾噘烻ᓧ鈠ᴥ徰穆ꘛ蹕綻表鯍裊' + 14 | '鮕漨踒ꠍ픸Ä☶莒浏钸목탬툖氭ˠٸ൪㤌ᶟ訧ᜒೳ揪Ⴛ摖㸣᳑⹞걀ꢢ䏹ῖ"'; 15 | 16 | describe('routesUtils.isValidObjectKey', () => { 17 | it('should return isValid false if object key name starts with a ' + 18 | 'blacklisted prefix', () => { 19 | const result = routesUtils.isValidObjectKey('bannedkey', [bannedStr]); 20 | // return { isValid: false, invalidPrefix }; 21 | assert.strictEqual(result.isValid, false); 22 | assert.strictEqual(result.invalidPrefix, bannedStr); 23 | }); 24 | 25 | it('should return isValid false if object key name exceeds length of 915', 26 | () => { 27 | const key = 'a'.repeat(916); 28 | const result = routesUtils.isValidObjectKey(key, prefixBlacklist); 29 | assert.strictEqual(result.isValid, false); 30 | }); 31 | 32 | it('should return isValid true for a utf8 string of byte size 915', () => { 33 | const result = routesUtils.isValidObjectKey(keyutf8, prefixBlacklist); 34 | assert.strictEqual(result.isValid, true); 35 | }); 36 | }); 37 | -------------------------------------------------------------------------------- /tests/unit/s3routes/routesUtils/okHeaderResponse.spec.js: -------------------------------------------------------------------------------- 1 | const sinon = require('sinon'); 2 | const { routesUtils } = require('../../../../lib/s3routes'); 3 | const werelogs = require('werelogs'); 4 | const assert = require('assert'); 5 | 6 | const logger = new werelogs.Logger('ErrorHtmlResponse', 'debug', 'debug'); 7 | const log = logger.newRequestLogger(); 8 | 9 | describe('routesutils.okHeaderResponse', () => { 10 | it('should return 200 status code and no body', done => { 11 | const headers = { 12 | 'x-amz-request-id': 'request-id', 13 | 'x-amz-id-2': 'request-id', 14 | }; 15 | const httpCode = 200; 16 | const res = { 17 | setHeader: sinon.stub(), 18 | writeHead: sinon.stub(), 19 | end: () => { 20 | assert.strictEqual(res.setHeader.callCount, 5); 21 | assert.strictEqual(res.writeHead.calledWith(httpCode), true); 22 | done(); 23 | }, 24 | }; 25 | routesUtils.okHeaderResponse(headers, res, httpCode, log); 26 | }); 27 | }); 28 | -------------------------------------------------------------------------------- /tests/unit/s3routes/routesUtils/toArsenalError.spec.ts: -------------------------------------------------------------------------------- 1 | import { ArsenalError, errorInstances } from '../../../../lib/errors'; 2 | import { toArsenalError } from '../../../../lib/s3routes/routesUtils'; 3 | 4 | describe('toArsenalError', () => { 5 | it('should return the same error if input is already an ArsenalError', () => { 6 | const originalError = errorInstances.NoSuchBucket; 7 | const result = toArsenalError(originalError); 8 | expect(result).toBe(originalError); 9 | }); 10 | 11 | it('should use existing error instance if error message matches known error type', () => { 12 | const error = new Error('NoSuchBucket'); 13 | const result = toArsenalError(error); 14 | expect(result).toBe(errorInstances.NoSuchBucket); 15 | }); 16 | 17 | it('should wrap unknown error in InternalError with custom description', () => { 18 | const error = new Error('Unknown error occurred'); 19 | const result = toArsenalError(error); 20 | expect(result).toBeInstanceOf(ArsenalError); 21 | expect(result.message).toBe('InternalError'); 22 | expect(result.description).toBe('Unknown error occurred'); 23 | }); 24 | }); 25 | -------------------------------------------------------------------------------- /tests/unit/shuffle.spec.ts: -------------------------------------------------------------------------------- 1 | import { shuffle } from '../../index'; 2 | 3 | describe('Shuffle', () => { 4 | test('should not lose data', () => { 5 | const base = new Array(1000).fill(null).map((_, i) => i); 6 | const comp = [...base]; 7 | shuffle(comp); 8 | expect(comp.length).toEqual(base.length); 9 | const incs = comp.reduce((acc, val) => acc && base.includes(val), true); 10 | expect(incs).toBeTruthy(); 11 | }); 12 | test('should correctly handle empty array', () => { 13 | const base = []; 14 | shuffle(base); 15 | expect(base).toEqual([]); 16 | }); 17 | test('should correctly handle one-element array', () => { 18 | const base = [1]; 19 | shuffle(base); 20 | expect(base).toEqual([1]); 21 | }); 22 | }); 23 | -------------------------------------------------------------------------------- /tests/unit/storage/data/DummyObjectStream.spec.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const DummyObjectStream = require('./DummyObjectStream'); 3 | 4 | async function testStream(startByteOffset, streamSize, expectedData) { 5 | const p = new Promise((resolve, reject) => { 6 | const dos = new DummyObjectStream(startByteOffset, streamSize); 7 | const readChunks = []; 8 | dos 9 | .on('data', chunk => readChunks.push(chunk)) 10 | .on('error', err => reject(err)) 11 | .on('end', () => { 12 | assert.strictEqual(readChunks.join(''), expectedData); 13 | resolve(); 14 | }); 15 | }); 16 | return p; 17 | } 18 | 19 | describe('DummyObjectStream', () => { 20 | jest.setTimeout(30000); 21 | it('should return a stream of 8-byte hex-encoded blocks', async () => { 22 | /* eslint-disable no-unused-expressions */ 23 | await testStream(0, 0, ''); 24 | await testStream(50, 0, ''); 25 | await testStream(0, 1, ' '); 26 | await testStream(1, 1, '0'); 27 | await testStream(1, 7, '0000000'); 28 | await testStream(0, 8, ' 0000000'); 29 | await testStream(1, 8, '0000000 '); 30 | await testStream(0, 10, ' 0000000 0'); 31 | await testStream(1, 10, '0000000 00'); 32 | await testStream(7, 5, '0 000'); 33 | await testStream(7, 12, '0 0000008 00'); 34 | await testStream(8, 12, ' 0000008 000'); 35 | await testStream(9, 12, '0000008 0000'); 36 | await testStream(40, 16, ' 0000028 0000030'); 37 | // check that offsets wrap around after 256MB 38 | await testStream(256 * 1024 * 1024 - 8, 16, ' ffffff8 0000000'); 39 | await testStream(567890123, 30, '950c8 1d950d0 1d950d8 1d950e0 '); 40 | 41 | // test larger streams with 8MiB of contents 42 | const expectedLarge = 43 | new Array(1024 * 1024).fill() 44 | .map((x, i) => ` ${`000000${Number(i * 8).toString(16)}`.slice(-7)}`) 45 | .join(''); 46 | await testStream(0, 8 * 1024 * 1024, expectedLarge); 47 | 48 | const expectedLarge2 = 49 | ['950c8'] 50 | .concat(new Array(1024 * 1024).fill() 51 | .map((x, i) => ` ${Number(0x1d950d0 + i * 8).toString(16)}`)) 52 | .concat([' 25']) 53 | .join(''); 54 | await testStream(567890123, 5 + 8 * 1024 * 1024 + 3, expectedLarge2); 55 | /* eslint-enable no-unused-expressions */ 56 | }); 57 | }); 58 | -------------------------------------------------------------------------------- /tests/unit/storage/data/mockClients/MockAzureClient.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const http = require('http'); 3 | const MetadataWrapper = 4 | require('../../../../../lib/storage/metadata/MetadataWrapper'); 5 | 6 | class MockAzureClient { 7 | put(stream, size, keyContext, reqUids, callback, skey, metadata) { 8 | if (stream) { 9 | assert(stream instanceof http.IncomingMessage); 10 | } 11 | assert.strictEqual(typeof size, 'number'); 12 | assert.strictEqual(typeof keyContext, 'object'); 13 | const { objectKey, bucketName } = keyContext; 14 | assert.strictEqual(typeof objectKey, 'string'); 15 | assert.strictEqual(typeof bucketName, 'string'); 16 | assert.strictEqual(typeof reqUids, 'string'); 17 | assert.strictEqual(typeof callback, 'function'); 18 | assert.equal(skey, null); 19 | assert(metadata instanceof MetadataWrapper); 20 | return callback(null, objectKey); 21 | } 22 | 23 | get(objectGetInfo, range, reqUids, callback) { 24 | assert.strictEqual(typeof objectGetInfo, 'object'); 25 | const { key, bucketName } = objectGetInfo; 26 | if (range) { 27 | assert(Array.isArray(range)); 28 | } 29 | assert.strictEqual(typeof key, 'string'); 30 | assert.strictEqual(typeof bucketName, 'string'); 31 | assert.strictEqual(typeof reqUids, 'string'); 32 | assert.strictEqual(typeof callback, 'function'); 33 | return callback(); 34 | } 35 | 36 | delete(objectGetInfo, reqUids, callback) { 37 | assert.strictEqual(typeof objectGetInfo, 'object'); 38 | const { key, bucketName } = objectGetInfo; 39 | assert.strictEqual(typeof key, 'string'); 40 | assert.strictEqual(typeof bucketName, 'string'); 41 | assert.strictEqual(typeof reqUids, 'string'); 42 | assert.strictEqual(typeof callback, 'function'); 43 | return callback(); 44 | } 45 | } 46 | 47 | module.exports = MockAzureClient; 48 | -------------------------------------------------------------------------------- /tests/unit/storage/data/mockClients/MockSproxydClient.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const crypto = require('crypto'); 3 | const http = require('http'); 4 | 5 | function genSproxydKey() { 6 | return crypto.randomBytes(20).toString('hex'); 7 | } 8 | 9 | class MockSproxydclient { 10 | put(stream, size, params, reqUids, callback, keyScheme) { 11 | if (keyScheme) { 12 | assert.strictEqual(typeof keyScheme, 'string'); 13 | assert.strictEqual(keyScheme.length, 40); 14 | } 15 | if (stream) { 16 | assert(stream instanceof http.IncomingMessage); 17 | } 18 | assert.strictEqual(typeof size, 'number'); 19 | assert.strictEqual(typeof params, 'object'); 20 | 21 | const { bucketName, objectKey } = params; 22 | assert.strictEqual(typeof bucketName, 'string'); 23 | assert.strictEqual(typeof objectKey, 'string'); 24 | assert.strictEqual(typeof reqUids, 'string'); 25 | assert.strictEqual(typeof callback, 'function'); 26 | return callback(null, genSproxydKey()); 27 | } 28 | 29 | get(key, range, reqUids, callback) { 30 | assert.strictEqual(typeof key, 'string'); 31 | assert.strictEqual(key.length, 40); 32 | if (range) { 33 | assert(Array.isArray(range)); 34 | } 35 | assert.strictEqual(typeof reqUids, 'string'); 36 | assert.strictEqual(typeof callback, 'function'); 37 | return callback(); 38 | } 39 | 40 | delete(key, reqUids, callback) { 41 | assert.strictEqual(typeof key, 'string'); 42 | assert.strictEqual(key.length, 40); 43 | assert.strictEqual(typeof reqUids, 'string'); 44 | assert.strictEqual(typeof callback, 'function'); 45 | return callback(); 46 | } 47 | } 48 | 49 | module.exports = MockSproxydclient; 50 | -------------------------------------------------------------------------------- /tests/unit/storage/metadata/in_memory/bucket_utilities.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict'; //eslint-disable-line 2 | 3 | const assert = require('assert'); 4 | 5 | const { markerFilterMPU } = 6 | require('../../../../../lib/storage/metadata/in_memory/bucket_utilities'); 7 | 8 | function dupeArray(arr) { 9 | const dupe = []; 10 | 11 | arr.forEach(i => { 12 | dupe.push(Object.assign({}, i)); 13 | }); 14 | 15 | return dupe; 16 | } 17 | 18 | describe('bucket utility methods for in_memory backend', () => { 19 | it('should return an array of multipart uploads starting with the item ' + 20 | 'right after the specified keyMarker and uploadIdMarker', () => { 21 | const mpus = [ 22 | { 23 | key: 'key-1', 24 | uploadId: '2624ca6080c841d48a2481941df868a9', 25 | }, 26 | { 27 | key: 'key-1', 28 | uploadId: '4ffeca96b0c24ea9b538b8f0b60cede3', 29 | }, 30 | { 31 | key: 'key-1', 32 | uploadId: '52e5b94474894990a2b94330bb3c8fa9', 33 | }, 34 | { 35 | key: 'key-1', 36 | uploadId: '54e530c5d4c741898c8e161d426591cb', 37 | }, 38 | { 39 | key: 'key-1', 40 | uploadId: '6cc59f9d29254e81ab6cb6332fb46314', 41 | }, 42 | { 43 | key: 'key-1', 44 | uploadId: 'fe9dd10776c9476697632d0b55960a05', 45 | }, 46 | { 47 | key: 'key-2', 48 | uploadId: '68e24ccb96c14beea79bf01fc130fdf5', 49 | }, 50 | ]; 51 | 52 | [ 53 | { 54 | keyMarker: 'key-1', 55 | uploadIdMarker: '54e530c5d4c741898c8e161d426591cb', 56 | expected: 3, 57 | }, 58 | { 59 | keyMarker: 'key-2', 60 | uploadIdMarker: '68e24ccb96c14beea79bf01fc130fdf5', 61 | expected: 0, 62 | }, 63 | { 64 | keyMarker: 'key-1', 65 | uploadIdMarker: '2624ca6080c841d48a2481941df868a9', 66 | expected: 6, 67 | }, 68 | ].forEach(item => { 69 | const res = markerFilterMPU(item, dupeArray(mpus)); 70 | assert.equal(res.length, item.expected); 71 | 72 | const expected = mpus.slice(mpus.length - res.length); 73 | assert.deepStrictEqual(res, expected); 74 | }); 75 | }); 76 | }); 77 | -------------------------------------------------------------------------------- /tests/unit/storage/metadata/mongoclient/listObject.spec.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const werelogs = require('werelogs'); 3 | const logger = new werelogs.Logger('MongoClientInterface', 'debug', 'debug'); 4 | const errors = require('../../../../../lib/errors').default; 5 | const sinon = require('sinon'); 6 | const MongoClientInterface = 7 | require('../../../../../lib/storage/metadata/mongoclient/MongoClientInterface'); 8 | 9 | describe('MongoClientInterface:listObject', () => { 10 | let client; 11 | 12 | beforeAll(done => { 13 | client = new MongoClientInterface({}); 14 | return done(); 15 | }); 16 | 17 | afterEach(done => { 18 | sinon.restore(); 19 | return done(); 20 | }); 21 | 22 | it('should fail when getBucketVFormat fails', done => { 23 | sinon.stub(client, 'getCollection').callsFake(() => ({})); 24 | sinon.stub(client, 'getBucketVFormat').callsFake((bucketName, log, cb) => cb(errors.InternalError)); 25 | client.listObject('example-bucket', { listingType: 'DelimiterMaster' }, logger, err => { 26 | assert.deepStrictEqual(err, errors.InternalError); 27 | return done(); 28 | }); 29 | }); 30 | 31 | it('should fail when internalListObject fails', done => { 32 | sinon.stub(client, 'getCollection').callsFake(() => ({})); 33 | sinon.stub(client, 'getBucketVFormat').callsFake((bucketName, log, cb) => cb(null, 'v0')); 34 | sinon.stub(client, 'internalListObject').callsFake((...args) => args[5](errors.InternalError)); 35 | client.listObject('example-bucket', { listingType: 'DelimiterMaster' }, logger, err => { 36 | assert.deepStrictEqual(err, errors.InternalError); 37 | return done(); 38 | }); 39 | }); 40 | }); 41 | -------------------------------------------------------------------------------- /tests/unit/storage/metadata/mongoclient/utils/DummyConfigObject.js: -------------------------------------------------------------------------------- 1 | const { EventEmitter } = require('events'); 2 | 3 | class DummyConfigObject extends EventEmitter { 4 | constructor() { 5 | super(); 6 | this.locationConstraints = null; 7 | this.isTest = true; 8 | } 9 | 10 | setLocationConstraints(locationConstraints) { 11 | this.locationConstraints = locationConstraints; 12 | this.emit('location-constraints-update'); 13 | } 14 | } 15 | 16 | module.exports = DummyConfigObject; 17 | -------------------------------------------------------------------------------- /tests/unit/storage/metadata/mongoclient/utils/DummyRequestLogger.js: -------------------------------------------------------------------------------- 1 | class DummyRequestLogger { 2 | constructor() { 3 | this.ops = []; 4 | this.counts = { 5 | trace: 0, 6 | debug: 0, 7 | info: 0, 8 | warn: 0, 9 | error: 0, 10 | fatal: 0, 11 | }; 12 | this.defaultFields = {}; 13 | } 14 | 15 | trace(msg) { 16 | this.ops.push(['trace', [msg]]); 17 | this.counts.trace += 1; 18 | } 19 | 20 | debug(msg) { 21 | this.ops.push(['debug', [msg]]); 22 | this.counts.debug += 1; 23 | } 24 | 25 | info(msg) { 26 | this.ops.push(['info', [msg]]); 27 | this.counts.info += 1; 28 | } 29 | 30 | warn(msg) { 31 | this.ops.push(['warn', [msg]]); 32 | this.counts.warn += 1; 33 | } 34 | 35 | error(msg) { 36 | this.ops.push(['error', [msg]]); 37 | this.counts.error += 1; 38 | } 39 | 40 | fatal(msg) { 41 | this.ops.push(['fatal', [msg]]); 42 | this.counts.fatal += 1; 43 | } 44 | 45 | getSerializedUids() { // eslint-disable-line class-methods-use-this 46 | return 'dummy:Serialized:Uids'; 47 | } 48 | 49 | addDefaultFields(fields) { 50 | Object.assign(this.defaultFields, fields); 51 | } 52 | 53 | end() { 54 | return this; 55 | } 56 | } 57 | 58 | module.exports = DummyRequestLogger; 59 | -------------------------------------------------------------------------------- /tests/unit/storage/metadata/mongoclient/utils/helper.js: -------------------------------------------------------------------------------- 1 | const basicMD = { 2 | 'content-length': 0, 3 | 'key': '', 4 | 'versionId': '', 5 | 'replicationInfo': { 6 | backends: [], // site, status 7 | }, 8 | 'dataStoreName': 'mongotest', 9 | }; 10 | 11 | function generateMD(objKey, size, versionId, repBackends) { 12 | const retMD = JSON.parse(JSON.stringify(basicMD)); 13 | retMD.key = objKey; 14 | retMD['content-length'] = size; 15 | retMD.versionId = versionId; 16 | if (repBackends && Array.isArray(repBackends)) { 17 | retMD.replicationInfo.backends.push(...repBackends); 18 | } 19 | return retMD; 20 | } 21 | 22 | module.exports = { 23 | generateMD, 24 | }; 25 | -------------------------------------------------------------------------------- /tests/unit/stream/readJSONStreamObject.spec.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const stream = require('stream'); 3 | const joi = require('joi'); 4 | const readJSONStreamObject = require('../../../lib/stream/readJSONStreamObject').default; 5 | 6 | class ReqStream extends stream.Readable { 7 | constructor(contents) { 8 | super(); 9 | this.contents = contents; 10 | } 11 | 12 | _read() { 13 | while (this.contents.length > 0) { 14 | this.push(this.contents.slice(0, 1000)); 15 | this.contents = this.contents.slice(1000); 16 | } 17 | this.push(null); 18 | } 19 | } 20 | 21 | describe('readJSONStreamObject', () => { 22 | [{ 23 | desc: 'accept a valid JSON object', 24 | contents: '{"foo":"bar","baz":42}', 25 | error: false, 26 | value: { foo: 'bar', baz: 42 }, 27 | }, { 28 | desc: 'error with empty stream contents', 29 | contents: '', 30 | error: true, 31 | }, { 32 | desc: 'error if stream contents does not match against the validation schema', 33 | contents: '"foo"', 34 | joiSchema: joi.object(), 35 | error: true, 36 | }, { 37 | desc: 'accept a large JSON payload', 38 | contents: `[${new Array(10000).fill('"some-payload"').join(',')}]`, 39 | error: false, 40 | value: new Array(10000).fill('some-payload'), 41 | }].forEach(testCase => { 42 | it(`should ${testCase.desc}`, async () => { 43 | let value; 44 | try { 45 | value = await readJSONStreamObject( 46 | new ReqStream(testCase.contents), testCase.joiSchema); 47 | } catch (err) { 48 | assert.strictEqual(testCase.error, true); 49 | return undefined; 50 | } 51 | assert.strictEqual(testCase.error, false); 52 | assert.deepStrictEqual(testCase.value, value); 53 | return undefined; 54 | }); 55 | }); 56 | }); 57 | -------------------------------------------------------------------------------- /tests/unit/stringHash.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict';// eslint-disable-line strict 2 | 3 | const assert = require('assert'); 4 | const crypto = require('crypto'); 5 | 6 | const stringHash = require('../../index').stringHash; 7 | 8 | const ARRAY_LENGTH = 1000; 9 | const STRING_COUNT = 1000000; 10 | const ERROR = 20; 11 | 12 | 13 | function randomString(length) { 14 | return crypto.randomBytes(Math.ceil(length / 2)) 15 | .toString('hex') 16 | .slice(0, length); 17 | } 18 | 19 | function check(array) { 20 | const x = STRING_COUNT / ARRAY_LENGTH; 21 | const error = x * (ERROR / 100); 22 | const min = x - error; 23 | const max = x + error; 24 | return !array.every(e => e >= min && e <= max); 25 | } 26 | 27 | describe('StringHash', () => { 28 | it('Should compute a string hash', done => { 29 | const hash1 = stringHash('Hello!'); 30 | const hash2 = stringHash('Hello?'); 31 | assert.notDeepStrictEqual(hash1, hash2); 32 | done(); 33 | }); 34 | it(`Should distribute uniformly with a maximum of ${ERROR}% of deviation`, 35 | (done) => { 36 | jest.setTimeout(20000); 37 | const strings = new Array(STRING_COUNT).fill('') 38 | .map(() => randomString(10)); 39 | const arr = new Array(ARRAY_LENGTH).fill(0); 40 | strings.forEach(string => { 41 | const ind = stringHash(string) % ARRAY_LENGTH; 42 | ++arr[ind]; 43 | }); 44 | done(check(arr)); 45 | }); 46 | }); 47 | -------------------------------------------------------------------------------- /tests/utils/DummyRequest.js: -------------------------------------------------------------------------------- 1 | const http = require('http'); 2 | 3 | class DummyRequest extends http.IncomingMessage { 4 | constructor(obj, msg) { 5 | super(); 6 | Object.keys(obj).forEach(x => { 7 | this[x] = obj[x]; 8 | }); 9 | const contentLength = this.headers['content-length']; 10 | if (this.parsedContentLength === undefined) { 11 | if (contentLength !== undefined) { 12 | this.parsedContentLength = parseInt(contentLength, 10); 13 | } else if (msg !== undefined) { 14 | this.parsedContentLength = msg.length; 15 | } else { 16 | this.parsedContentLength = 0; 17 | } 18 | } 19 | 20 | if (Array.isArray(msg)) { 21 | msg.forEach(part => { 22 | this.push(part); 23 | }); 24 | } else { 25 | this.push(msg); 26 | } 27 | this.push(null); 28 | } 29 | } 30 | 31 | module.exports = DummyRequest; 32 | -------------------------------------------------------------------------------- /tests/utils/ca.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDfDCCAmSgAwIBAgIJAKFG6vMw+4JHMA0GCSqGSIb3DQEBCwUAMFQxCzAJBgNV 3 | BAYTAkZSMQ8wDQYDVQQIDAZGcmFuY2UxDjAMBgNVBAcMBVBhcmlzMRAwDgYDVQQK 4 | DAdTY2FsaXR5MRIwEAYDVQQDDAkxMjcuMC4wLjEwIBcNMTYwODA5MjEyODU1WhgP 5 | MjI5MDA1MjQyMTI4NTVaMFQxCzAJBgNVBAYTAkZSMQ8wDQYDVQQIDAZGcmFuY2Ux 6 | DjAMBgNVBAcMBVBhcmlzMRAwDgYDVQQKDAdTY2FsaXR5MRIwEAYDVQQDDAkxMjcu 7 | MC4wLjEwggEhMA0GCSqGSIb3DQEBAQUAA4IBDgAwggEJAoIBAC/KiuFtQEnmLjVN 8 | yfug4l04buKmQG/XROz/2ykS8gQdcrao9sFfLiBcuzoC4vOJpRq+q147dg8III1f 9 | LpDq2OhihoJeIoFGC6t0uo3UHhgc/1W5YFaYRW28V8gasG9ZADFI7ASNiiaKCHCJ 10 | u1jS3CQJF9Rwd3BYA7+CC0iP2sl0cLwNmd3XHIcqhmSDWushU+iO5jemkBGkZDWt 11 | AEdRw4ghpyKPgRg2WkTErN+Ic/R0pa1ZFmUHbYyR5GCQe9ewi2nKXwmJP3oByVE0 12 | s7Dqt5Vurq0pFSrRFCAr5taTachRT9GyU0tl71rQKPP91VLVlF2TQO8Psya94hw7 13 | zZRw4sUCAwEAAaNQME4wHQYDVR0OBBYEFG8o69lTHam/X0ljn60eCh9OeCZ5MB8G 14 | A1UdIwQYMBaAFG8o69lTHam/X0ljn60eCh9OeCZ5MAwGA1UdEwQFMAMBAf8wDQYJ 15 | KoZIhvcNAQELBQADggEBACNgYHK28e2kKtrlm9q6K7mMyHsRQifqWztjhmOU4iaU 16 | 7cnO1pdQy2vgjJYkJz7L09A183OhasgMjDjBmF17SdlJADKQV0ibHlhWbTtL+bBy 17 | CsPUjKtD3XA6CkdBnslDceHahO2nvhbDKi6A4gw4cz3/CZKPW1supbuUFfw5uqZV 18 | 6CJGyOixIQJPSWIVsdmVOEu207DTVtCDrrYIMnITRoTaXF0o8rbwCKrTPsxtbJ2k 19 | vtZFJu9VKe52gszx5VfeIIJRNyyBFCFYZgoHhj+2vYJNg3Evkk12itltJ+sOtH2C 20 | zFSVtwyXsT1KoBuKgdr/HKWLsjGuLW7hlK4oe7qeL/A= 21 | -----END CERTIFICATE----- 22 | -------------------------------------------------------------------------------- /tests/utils/dummyS3Config.json: -------------------------------------------------------------------------------- 1 | { 2 | "port": 8000, 3 | "listenOn": [], 4 | "replicationGroupId": "RG001", 5 | "restEndpoints": { 6 | "localhost": "us-east-1", 7 | "127.0.0.1": "us-east-1", 8 | "cloudserver-front": "us-east-1", 9 | "s3.docker.test": "us-east-1", 10 | "127.0.0.2": "us-east-1", 11 | "s3.amazonaws.com": "us-east-1" 12 | }, 13 | "websiteEndpoints": ["s3-website-us-east-1.amazonaws.com", 14 | "s3-website.us-east-2.amazonaws.com", 15 | "s3-website-us-west-1.amazonaws.com", 16 | "s3-website-us-west-2.amazonaws.com", 17 | "s3-website.ap-south-1.amazonaws.com", 18 | "s3-website.ap-northeast-2.amazonaws.com", 19 | "s3-website-ap-southeast-1.amazonaws.com", 20 | "s3-website-ap-southeast-2.amazonaws.com", 21 | "s3-website-ap-northeast-1.amazonaws.com", 22 | "s3-website.eu-central-1.amazonaws.com", 23 | "s3-website-eu-west-1.amazonaws.com", 24 | "s3-website-sa-east-1.amazonaws.com", 25 | "s3-website.localhost", 26 | "s3-website.scality.test"], 27 | "replicationEndpoints": [{ 28 | "site": "zenko", 29 | "servers": ["127.0.0.1:8000"], 30 | "default": true 31 | }, { 32 | "site": "us-east-2", 33 | "type": "aws_s3" 34 | }], 35 | "cdmi": { 36 | "host": "localhost", 37 | "port": 81, 38 | "path": "/dewpoint", 39 | "readonly": true 40 | }, 41 | "bucketd": { 42 | "bootstrap": ["localhost:9000"] 43 | }, 44 | "vaultd": { 45 | "host": "localhost", 46 | "port": 8500 47 | }, 48 | "clusters": 10, 49 | "log": { 50 | "logLevel": "info", 51 | "dumpLevel": "error" 52 | }, 53 | "healthChecks": { 54 | "allowFrom": ["127.0.0.1/8", "::1"] 55 | }, 56 | "metadataClient": { 57 | "host": "localhost", 58 | "port": 9990 59 | }, 60 | "dataClient": { 61 | "host": "localhost", 62 | "port": 9991 63 | }, 64 | "metadataDaemon": { 65 | "bindAddress": "localhost", 66 | "port": 9990 67 | }, 68 | "dataDaemon": { 69 | "bindAddress": "localhost", 70 | "port": 9991 71 | }, 72 | "recordLog": { 73 | "enabled": false, 74 | "recordLogName": "s3-recordlog" 75 | }, 76 | "mongodb": { 77 | "host": "localhost", 78 | "port": 27018, 79 | "database": "metadata" 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /tests/utils/performListing.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | 3 | module.exports = function performListing(data, Extension, params, logger, vFormat) { 4 | const listing = new Extension(params, logger, vFormat); 5 | const mdParams = listing.genMDParams(); 6 | assert.strictEqual(typeof mdParams, 'object'); 7 | data.every(e => listing.filter(e) >= 0); 8 | return listing.result(); 9 | }; 10 | -------------------------------------------------------------------------------- /tests/utils/test.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDJTCCAg0CCQDIQOzoZwGjgzANBgkqhkiG9w0BAQsFADBUMQswCQYDVQQGEwJG 3 | UjEPMA0GA1UECAwGRnJhbmNlMQ4wDAYDVQQHDAVQYXJpczEQMA4GA1UECgwHU2Nh 4 | bGl0eTESMBAGA1UEAwwJMTI3LjAuMC4xMCAXDTE2MDgwOTIxMjg1NVoYDzIyOTAw 5 | NTI0MjEyODU1WjBUMQswCQYDVQQGEwJGUjEPMA0GA1UECAwGRnJhbmNlMQ4wDAYD 6 | VQQHDAVQYXJpczEQMA4GA1UECgwHU2NhbGl0eTESMBAGA1UEAwwJMTI3LjAuMC4x 7 | MIIBITANBgkqhkiG9w0BAQEFAAOCAQ4AMIIBCQKCAQA8tCU8il2I5ltNXj+dDOwV 8 | 5VC5yV9KHXVhqognpFaTW3NUHScABOEqxVqBL7dPBxWEEglGf3fd2fTnVDu+rWK2 9 | 3LwGlwbKCWV7vnTp+3hsG4J5d6ZGwdQ72b8P29q+gtz7nNhQ7wm+nqf6LQgJsZUN 10 | fK+S5bhunShCOwYA5m+LTVg9likipTb8wjqHAMsokStkP+hSyvQIH7NvnRVl025a 11 | OfvjdHBtKhcdKDQU96Jt0sz1ET52t/CrPIvL9BK3gclPPKdD0S0zj1xagYPrYcLk 12 | bwK1246Lq61S+vwMN8gtdUYNMSp4tKUVa9wvo3KMf5TNom+x8oZe1MQKZ5AZgL1d 13 | AgMBAAEwDQYJKoZIhvcNAQELBQADggEBAA2GkVkqBaXs8Lg89q/177Zz8xK+D+Qu 14 | i26pFYpzIWZg2DrzgtyUbQ4FmqPfugDhVnL9oSLxqiXnPcAVBn93U/iDcb2h+B20 15 | 2onLnPA6OE4Vl/yEcFQ1w0+5UO5d2dyzrZOYfx/QestIGTJByZn2hknMMX9k75p8 16 | TT0v0ml3HZw2m6RHgmt2hVI80lNgX9ceA37TElWrSleagMbDXoviqmAJrp8zJSnY 17 | Ex0EO2Zg+Slq/03jJ+MoIXlXAbDihoNJquaK1F/n+JsLdfiEOAVgU1ZlWV+NlQgF 18 | YuwNv5PG6apI0U/tnix2D32h/V4IeP1lodP73R0CzIQC61OCu/nPG4g= 19 | -----END CERTIFICATE----- 20 | -------------------------------------------------------------------------------- /tests/utils/test.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEnwIBAAKCAQA8tCU8il2I5ltNXj+dDOwV5VC5yV9KHXVhqognpFaTW3NUHScA 3 | BOEqxVqBL7dPBxWEEglGf3fd2fTnVDu+rWK23LwGlwbKCWV7vnTp+3hsG4J5d6ZG 4 | wdQ72b8P29q+gtz7nNhQ7wm+nqf6LQgJsZUNfK+S5bhunShCOwYA5m+LTVg9liki 5 | pTb8wjqHAMsokStkP+hSyvQIH7NvnRVl025aOfvjdHBtKhcdKDQU96Jt0sz1ET52 6 | t/CrPIvL9BK3gclPPKdD0S0zj1xagYPrYcLkbwK1246Lq61S+vwMN8gtdUYNMSp4 7 | tKUVa9wvo3KMf5TNom+x8oZe1MQKZ5AZgL1dAgMBAAECggEAHUpzmUXOoks9DRUI 8 | LvjYRixzimIvl6ylQi4uKqqyl8IN4S177xdrqt61sBZdJkXtzN+DGEBTj3onISRU 9 | +8ngEwpps+hV/1EPZeldsrWDwu53Q7NHSWcnyIVmGvnkD/9HMCXbfxoIorEdrqrh 10 | 8QN+P5bFhWVRXBZ7IZIncHHAqv973eehQr73E6E5ISGYO47LDp5g3v8uOAsgS1u3 11 | I2si0GA1Djgmk85pbeMJznj3/LPlmS8ZnKanKvLvpbDvj62m0+h3P9oJn/PogrLx 12 | hDM2Kq9+3mNapQgUUFLysTfJ6LcXZJafC4+slVSm3Rc/KjJI3hQ98vPchESnH6fa 13 | jT0JwQKBgH6my6X72oxL3PO3UF/vl0SuK0aKg8kfDmknr1a3nEcZXQTvrGAQM/KB 14 | tkImWpZ3nh5FMwdr8ogrCyOVn4NZhbZAnNcsEk5oYNS0mJ5W6g4Xk5CR+z6Ujqpb 15 | STQ3EIQkzweimtoRKHHoE59JR2NxE2M3OcO9MJf7cvamd1SkTQUxAoGAerMzqWpS 16 | SjznFXWCR8WNWGpgkhk7N5PvDVXHoLfte8oKAPqe0+YtnE+i4D9mJfVp03jB1ZNn 17 | FylircVMSNXTrKd6joJyyT5uaB6qLiEoyA9Y5TeBdWc8ik+TYeDOjCrmoi+ziIMV 18 | ETV6hz/1U59uJOhAkSaWCtm6GGpygFcyH+0CgYA2UTyn7DLX1rVlROpQqsW6a+Qd 19 | dcx9Vjds+9skGs0IZSVSt6O681gEGoNbLW3OeHC01MLE3RQAOE2nrkTiJWPGPUHG 20 | up0DSZq1vtpxlh79ejkMWL9jIH1rLIlhvnfz5IFx6df1zEQHThwURW47hMRm2cmJ 21 | XDtaAzpT9CLbhzeNEQKBgDUDFa/9kr4mYATCd15A+ReZJk4Y/p+9l4vgYtaKgN15 22 | 5iaUIWkVyuD8+zb9zUlbJbTLOJvpCqJULCE92/6f+8tdtLK7o5JVGeh8TzSM+Qyu 23 | rM6j05gA2YQ8a0Xflf2zT2AFUgEJ+WEtBNpIhNrzR+hEPBk6XZskhKWl4ACZK7vl 24 | AoGAKTTk0j+HgBfCLM+C31OFeUZTstAxdpgXOyuM6cWQ3JT+IbOkxW83mx6fetgQ 25 | uA8lcNqDQglHJNOU3bJiny1pRa+rvFkyfO7f4NKcaluIS1lAhGY1RY/osG6F2AdL 26 | URJdlTxpLQLq5FovZmI3pONphPn+s2Ywp2krylazHWI3kq8= 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2021", 4 | "module": "commonjs", 5 | "rootDir": "./", 6 | "resolveJsonModule": true, 7 | "allowJs": true, 8 | "checkJs": false, 9 | "outDir": "build", 10 | "esModuleInterop": true, 11 | "forceConsistentCasingInFileNames": true, 12 | "strict": true, 13 | "declaration": true, 14 | "noImplicitAny": false, 15 | "noEmitOnError": false, 16 | "sourceMap": true 17 | }, 18 | "include": ["index.ts", "lib"], 19 | "exclude": ["node_modules/*"], 20 | "compileOnSave": true 21 | } 22 | --------------------------------------------------------------------------------