├── .dockerignore ├── .github └── workflows │ ├── codeql-analysis.yml │ ├── dependency-review.yml │ └── shellsheck.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── build ├── .minio-dotnet │ └── install.sh ├── aws-sdk-go │ └── install.sh ├── aws-sdk-java-v2 │ ├── .gitattributes │ ├── .gitignore │ ├── app │ │ ├── build.gradle.kts │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── io │ │ │ └── minio │ │ │ └── awssdk │ │ │ └── v2 │ │ │ └── tests │ │ │ ├── FunctionalTests.java │ │ │ ├── LimitedInputStream.java │ │ │ ├── MintLogger.java │ │ │ ├── S3TestUtils.java │ │ │ └── Utils.java │ ├── gradle │ │ └── wrapper │ │ │ └── gradle-wrapper.properties │ ├── gradlew │ ├── gradlew.bat │ ├── install.sh │ └── settings.gradle.kts ├── aws-sdk-java │ ├── build.xml │ ├── install.sh │ ├── ivy.xml │ └── src │ │ ├── FunctionalTests.java │ │ ├── LimitedInputStream.java │ │ ├── MintLogger.java │ │ ├── S3TestUtils.java │ │ └── Utils.java ├── aws-sdk-php │ └── install.sh ├── aws-sdk-ruby │ └── install.sh ├── awscli │ ├── expect-100.patch │ └── install.sh ├── healthcheck │ └── install.sh ├── mc │ └── install.sh ├── minio-go │ └── install.sh ├── minio-java │ └── install.sh ├── minio-js │ └── install.sh ├── minio-py │ └── install.sh ├── s3cmd │ └── install.sh ├── s3select │ └── install.sh └── versioning │ ├── bucket.go │ ├── delete.go │ ├── get.go │ ├── go.mod │ ├── go.sum │ ├── install.sh │ ├── legalhold.go │ ├── list.go │ ├── main.go │ ├── put.go │ ├── retention.go │ ├── stat.go │ ├── tagging.go │ └── utils.go ├── create-data-files.sh ├── entrypoint.sh ├── install-packages.list ├── mint.sh ├── postinstall.sh ├── preinstall.sh ├── release.sh ├── remove-packages.list ├── run └── core │ ├── .minio-dotnet │ └── run.sh │ ├── aws-sdk-go │ ├── go.mod │ ├── go.sum │ ├── main.go │ └── run.sh │ ├── aws-sdk-java-v2 │ └── run.sh │ ├── aws-sdk-java │ └── run.sh │ ├── aws-sdk-php │ ├── README.md │ ├── composer.json │ ├── quick-tests.php │ └── run.sh │ ├── aws-sdk-ruby │ ├── README.md │ ├── aws-stub-tests.rb │ └── run.sh │ ├── awscli │ ├── README.md │ ├── run.sh │ └── test.sh │ ├── healthcheck │ ├── go.mod │ ├── go.sum │ ├── main.go │ └── run.sh │ ├── mc │ ├── README.md │ └── run.sh │ ├── minio-go │ ├── README.md │ ├── go.mod │ ├── go.sum │ └── run.sh │ ├── minio-java │ ├── README.md │ └── run.sh │ ├── minio-js │ ├── .mocharc.js │ ├── README.md │ ├── babel-register.js │ ├── minioreporter.js │ ├── package-lock.json │ ├── package.json │ └── run.sh │ ├── minio-py │ ├── README.md │ └── run.sh │ ├── s3cmd │ ├── .gitignore │ ├── README.md │ ├── run.sh │ └── test.sh │ ├── s3select │ ├── README.md │ ├── csv.py │ ├── run.sh │ ├── sql_ops.py │ ├── tests.py │ └── utils.py │ └── versioning │ └── run.sh └── source.sh /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | *.patch 3 | run/core/minio-go/main.go 4 | run/core/minio-go/minio-go 5 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ "master" ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ "master" ] 20 | schedule: 21 | - cron: '25 7 * * 0' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'go', 'javascript', 'python', 'ruby' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v3 42 | 43 | # Initializes the CodeQL tools for scanning. 44 | - name: Initialize CodeQL 45 | uses: github/codeql-action/init@v2 46 | with: 47 | languages: ${{ matrix.language }} 48 | # If you wish to specify custom queries, you can do so here or in a config file. 49 | # By default, queries listed here will override any specified in a config file. 50 | # Prefix the list here with "+" to use these queries and those in the config file. 51 | 52 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 53 | # queries: security-extended,security-and-quality 54 | 55 | 56 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 57 | # If this step fails, then you should remove it and run the build manually (see below) 58 | - name: Autobuild 59 | uses: github/codeql-action/autobuild@v2 60 | 61 | # ℹ️ Command-line programs to run using the OS shell. 62 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 63 | 64 | # If the Autobuild fails above, remove it and uncomment the following three lines. 65 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 66 | 67 | # - run: | 68 | # echo "Run, Build Application using script" 69 | # ./location_of_script_within_repo/buildscript.sh 70 | 71 | - name: Perform CodeQL Analysis 72 | uses: github/codeql-action/analyze@v2 73 | -------------------------------------------------------------------------------- /.github/workflows/dependency-review.yml: -------------------------------------------------------------------------------- 1 | # Dependency Review Action 2 | # 3 | # This Action will scan dependency manifest files that change as part of a Pull Request, surfacing known-vulnerable versions of the packages declared or updated in the PR. Once installed, if the workflow run is marked as required, PRs introducing known-vulnerable packages will be blocked from merging. 4 | # 5 | # Source repository: https://github.com/actions/dependency-review-action 6 | # Public documentation: https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/about-dependency-review#dependency-review-enforcement 7 | name: 'Dependency Review' 8 | on: [pull_request] 9 | 10 | permissions: 11 | contents: read 12 | 13 | jobs: 14 | dependency-review: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: 'Checkout Repository' 18 | uses: actions/checkout@v3 19 | - name: 'Dependency Review' 20 | uses: actions/dependency-review-action@v1 21 | -------------------------------------------------------------------------------- /.github/workflows/shellsheck.yml: -------------------------------------------------------------------------------- 1 | name: 'Trigger: Shell scripts formatting action' 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | sh-checker: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v1 13 | - name: Run the sh-checker 14 | uses: luizm/action-sh-checker@master 15 | env: 16 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 17 | SHFMT_OPTS: -s # arguments to shfmt. 18 | with: 19 | sh_checker_comment: true 20 | sh_checker_exclude: gradlew 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.test 2 | *.jar 3 | src/* 4 | temp 5 | __pycache__/ 6 | log/* 7 | minio.test 8 | bin/* 9 | node_modules 10 | # exception to the rule 11 | !log/.gitkeep 12 | !bin/.gitkeep 13 | *.class 14 | *~ 15 | run/core/minio-dotnet/bin/* 16 | run/core/minio-dotnet/obj/* 17 | run/core/minio-dotnet/out/* 18 | run/core/minio-go/main.go 19 | run/core/healthcheck/healthcheck 20 | .idea/ 21 | .phive/ 22 | .phplint.cache/ 23 | build/versioning/tests 24 | tools/ -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:22.04 2 | 3 | ENV DEBIAN_FRONTEND noninteractive 4 | ENV LANG C.UTF-8 5 | ENV GOROOT /usr/local/go 6 | ENV GOPATH /usr/local/gopath 7 | ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH 8 | ENV MINT_ROOT_DIR /mint 9 | 10 | RUN apt-get --yes update && apt-get --yes upgrade && \ 11 | apt-get --yes --quiet install wget jq curl git dnsmasq 12 | 13 | COPY . /mint 14 | 15 | WORKDIR /mint 16 | 17 | RUN /mint/create-data-files.sh 18 | RUN /mint/preinstall.sh 19 | RUN /mint/release.sh 20 | 21 | ENTRYPOINT ["/mint/entrypoint.sh"] 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Mint [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/mint.svg?maxAge=604800)](https://hub.docker.com/r/minio/mint/) 2 | 3 | Mint is a testing framework for Minio object server, available as a podman image. It runs correctness, benchmarking and stress tests. Following are the SDKs/tools used in correctness tests. 4 | 5 | - awscli 6 | - aws-sdk-go 7 | - aws-sdk-java 8 | - aws-sdk-java-v2 9 | - aws-sdk-php 10 | - aws-sdk-ruby 11 | - healthcheck 12 | - mc 13 | - minio-go 14 | - minio-java 15 | - minio-js 16 | - minio-py 17 | - s3cmd 18 | - s3select 19 | - versioning 20 | 21 | ## Running Mint 22 | 23 | Mint is run by `podman run` command which requires Podman to be installed. For Podman installation follow the steps [here](https://podman.io/getting-started/installation#installing-on-linux). 24 | 25 | To run Mint with Minio Play server as test target, 26 | 27 | ```sh 28 | $ podman run -e SERVER_ENDPOINT=play.minio.io:9000 -e ACCESS_KEY=Q3AM3UQ867SPQQA43P2F \ 29 | -e SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG -e ENABLE_HTTPS=1 minio/mint 30 | ``` 31 | 32 | After the tests are run, output is stored in `/mint/log` directory inside the container. To get these logs, use `podman cp` command. For example 33 | ```sh 34 | podman cp :/mint/log /tmp/logs 35 | ``` 36 | 37 | ### Mint environment variables 38 | 39 | Below environment variables are required to be passed to the podman container. Supported environment variables: 40 | 41 | | Environment variable | Description | Example | 42 | |:-----------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------| 43 | | `SERVER_ENDPOINT` | Endpoint of Minio server in the format `HOST:PORT`; for virtual style `IP:PORT` | `play.minio.io:9000` | 44 | | `ACCESS_KEY` | Access key for `SERVER_ENDPOINT` credentials | `Q3AM3UQ867SPQQA43P2F` | 45 | | `SECRET_KEY` | Secret Key for `SERVER_ENDPOINT` credentials | `zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG` | 46 | | `ENABLE_HTTPS` | (Optional) Set `1` to indicate to use HTTPS to access `SERVER_ENDPOINT`. Defaults to `0` (HTTP) | `1` | 47 | | `MINT_MODE` | (Optional) Set mode indicating what category of tests to be run by values `core`, `full`. Defaults to `core` | `full` | 48 | | `DOMAIN` | (Optional) Value of MINIO_DOMAIN environment variable used in Minio server | `myminio.com` | 49 | | `ENABLE_VIRTUAL_STYLE` | (Optional) Set `1` to indicate virtual style access . Defaults to `0` (Path style) | `1` | 50 | | `RUN_ON_FAIL` | (Optional) Set `1` to indicate execute all tests independent of failures (currently implemented for minio-go and minio-java) . Defaults to `0` | `1` | 51 | | `SERVER_REGION` | (Optional) Set custom region for region specific tests | `us-west-1` | 52 | 53 | ### Test virtual style access against Minio server 54 | 55 | To test Minio server virtual style access with Mint, follow these steps: 56 | 57 | - Set a domain in your Minio server using environment variable MINIO_DOMAIN. For example `export MINIO_DOMAIN=myminio.com`. 58 | - Start Minio server. 59 | - Execute Mint against Minio server (with `MINIO_DOMAIN` set to `myminio.com`) using this command 60 | ```sh 61 | $ podman run -e "SERVER_ENDPOINT=192.168.86.133:9000" -e "DOMAIN=minio.com" \ 62 | -e "ACCESS_KEY=minio" -e "SECRET_KEY=minio123" -e "ENABLE_HTTPS=0" \ 63 | -e "ENABLE_VIRTUAL_STYLE=1" minio/mint 64 | ``` 65 | 66 | ### Mint log format 67 | 68 | All test logs are stored in `/mint/log/log.json` as multiple JSON document. Below is the JSON format for every entry in the log file. 69 | 70 | | JSON field | Type | Description | Example | 71 | |:-----------|:---------|:--------------------------------------------------------------|:------------------------------------------------------| 72 | | `name` | _string_ | Testing tool/SDK name | `"aws-sdk-php"` | 73 | | `function` | _string_ | Test function name | `"getBucketLocation ( array $params = [] )"` | 74 | | `args` | _object_ | (Optional) Key/Value map of arguments passed to test function | `{"Bucket":"aws-sdk-php-bucket-20341"}` | 75 | | `duration` | _int_ | Time taken in milliseconds to run the test | `384` | 76 | | `status` | _string_ | one of `PASS`, `FAIL` or `NA` | `"PASS"` | 77 | | `alert` | _string_ | (Optional) Alert message indicating test failure | `"I/O error on create file"` | 78 | | `message` | _string_ | (Optional) Any log message | `"validating checksum of downloaded object"` | 79 | | `error` | _string_ | Detailed error message including stack trace on status `FAIL` | `"Error executing \"CompleteMultipartUpload\" on ...` | 80 | 81 | ## For Developers 82 | 83 | ### Running Mint development code 84 | 85 | After making changes to Mint source code a local podman image can be built/run by 86 | 87 | ```sh 88 | $ podman build -t minio/mint . -f Dockerfile 89 | $ podman run -e SERVER_ENDPOINT=play.minio.io:9000 -e ACCESS_KEY=Q3AM3UQ867SPQQA43P2F \ 90 | -e SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG \ 91 | -e ENABLE_HTTPS=1 -e MINT_MODE=full minio/mint:latest 92 | ``` 93 | 94 | 95 | ### Adding tests with new tool/SDK 96 | 97 | Below are the steps need to be followed 98 | 99 | - Create new app directory under [build](https://github.com/minio/mint/tree/master/build) and [run/core](https://github.com/minio/mint/tree/master/run/core) directories. 100 | - Create `install.sh` which does installation of required tool/SDK under app directory. 101 | - Any build and install time dependencies should be added to [install-packages.list](https://github.com/minio/mint/blob/master/install-packages.list). 102 | - Build time dependencies should be added to [remove-packages.list](https://github.com/minio/mint/blob/master/remove-packages.list) for removal to have clean Mint podman image. 103 | - Add `run.sh` in app directory under `run/core` which execute actual tests. 104 | 105 | #### Test data 106 | Tests may use pre-created data set to perform various object operations on Minio server. Below data files are available under `/mint/data` directory. 107 | 108 | | File name | Size | 109 | |:-----------------|:--------| 110 | | datafile-0-b | 0B | 111 | | datafile-1-b | 1B | 112 | | datafile-1-kB | 1KiB | 113 | | datafile-10-kB | 10KiB | 114 | | datafile-33-kB | 33KiB | 115 | | datafile-100-kB | 100KiB | 116 | | datafile-1-MB | 1MiB | 117 | | datafile-1.03-MB | 1.03MiB | 118 | | datafile-5-MB | 5MiB | 119 | | datafile-6-MB | 6MiB | 120 | | datafile-10-MB | 10MiB | 121 | | datafile-11-MB | 11MiB | 122 | | datafile-65-MB | 65MiB | 123 | | datafile-129-MB | 129MiB | 124 | 125 | ### Updating SDKs/binaries in the image 126 | 127 | In many cases, updating the SDKs or binaries in the image is just a matter of making a commit updating the corresponding version in this repo. However, in some cases, e.g. when `mc` needs to be updated (the latest `mc` is pulled in during each mint image build), a sort of "dummy" commit is required as an image rebuild must be triggered. Note that an empty commit does not appear to trigger the image rebuild in the Docker Hub. 128 | -------------------------------------------------------------------------------- /build/.minio-dotnet/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2017-2020 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | set -e 19 | 20 | MINIO_DOTNET_SDK_PATH="$MINT_RUN_CORE_DIR/minio-dotnet" 21 | 22 | MINIO_DOTNET_SDK_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/minio-dotnet/releases/latest | sed "s/https:\/\/github.com\/minio\/minio-dotnet\/releases\/tag\///") 23 | if [ -z "$MINIO_DOTNET_SDK_VERSION" ]; then 24 | echo "unable to get minio-dotnet version from github" 25 | exit 1 26 | fi 27 | 28 | out_dir="$MINIO_DOTNET_SDK_PATH/out" 29 | if [ -z "$out_dir" ]; then 30 | mkdir "$out_dir" 31 | fi 32 | 33 | temp_dir="$MINIO_DOTNET_SDK_PATH/temp" 34 | git clone --quiet https://github.com/minio/minio-dotnet.git "${temp_dir}" 35 | pushd "${temp_dir}" >/dev/null 36 | git checkout --quiet "tags/${MINIO_DOTNET_SDK_VERSION}" 37 | 38 | dotnet publish Minio.Functional.Tests --configuration Mint --framework net6.0 --output ../out 39 | 40 | popd >/dev/null 41 | rm -fr "${temp_dir}" 42 | -------------------------------------------------------------------------------- /build/aws-sdk-go/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | test_run_dir="$MINT_RUN_CORE_DIR/aws-sdk-go" 19 | (cd "$test_run_dir" && CGO_ENABLED=0 go build --ldflags "-s -w") 20 | -------------------------------------------------------------------------------- /build/aws-sdk-java-v2/.gitattributes: -------------------------------------------------------------------------------- 1 | # 2 | # https://help.github.com/articles/dealing-with-line-endings/ 3 | # 4 | # Linux start script should use lf 5 | /gradlew text eol=lf 6 | 7 | # These are Windows script files and should use crlf 8 | *.bat text eol=crlf 9 | 10 | -------------------------------------------------------------------------------- /build/aws-sdk-java-v2/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore Gradle project-specific cache directory 2 | .gradle 3 | .idea 4 | 5 | # Ignore Gradle build output directory 6 | app/build 7 | -------------------------------------------------------------------------------- /build/aws-sdk-java-v2/app/build.gradle.kts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file was generated by the Gradle 'init' task. 3 | * 4 | * This generated file contains a sample Java application project to get you started. 5 | * For more details on building Java & JVM projects, please refer to https://docs.gradle.org/8.5/userguide/building_java_projects.html in the Gradle documentation. 6 | */ 7 | import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar 8 | 9 | plugins { 10 | // Apply the application plugin to add support for building a CLI application in Java. 11 | application 12 | id("com.github.johnrengelman.shadow") version "8.1.1" 13 | } 14 | 15 | repositories { 16 | // Use Maven Central for resolving dependencies. 17 | mavenCentral() 18 | } 19 | 20 | dependencies { 21 | // AWS SDK dependencies 22 | implementation(platform("software.amazon.awssdk:bom:2.25.31")) 23 | implementation("software.amazon.awssdk:s3") 24 | implementation("software.amazon.awssdk:netty-nio-client") 25 | implementation("software.amazon.awssdk:aws-crt-client") 26 | 27 | // jackson dependency 28 | implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.13.+") 29 | } 30 | 31 | tasks.withType { 32 | archiveFileName.set("FunctionalTests.jar") 33 | } 34 | 35 | // Apply a specific Java toolchain to ease working on different environments. 36 | java { 37 | toolchain { 38 | languageVersion.set(JavaLanguageVersion.of(11)) 39 | } 40 | } 41 | 42 | application { 43 | // Define the main class for the application. 44 | mainClass.set("io.minio.awssdk.v2.tests.FunctionalTests") 45 | } 46 | -------------------------------------------------------------------------------- /build/aws-sdk-java-v2/app/src/main/java/io/minio/awssdk/v2/tests/LimitedInputStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Mint, (C) 2018-2023 Minio, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.minio.awssdk.v2.tests; 19 | 20 | import java.io.IOException; 21 | import java.io.InputStream; 22 | 23 | public class LimitedInputStream extends InputStream { 24 | private int skip; 25 | private int length; 26 | private InputStream is; 27 | 28 | LimitedInputStream(InputStream is, int skip, int length) { 29 | this.is = is; 30 | this.skip = skip; 31 | this.length = length; 32 | } 33 | 34 | @Override 35 | public int read() throws IOException { 36 | int r; 37 | while (skip > 0) { 38 | r = is.read(); 39 | if (r < 0) { 40 | throw new IOException("stream ended before being able to skip all bytes"); 41 | } 42 | skip--; 43 | } 44 | if (length == 0) { 45 | return -1; 46 | } 47 | r = is.read(); 48 | if (r < 0) { 49 | throw new IOException("stream ended before being able to read all bytes"); 50 | } 51 | length--; 52 | return r; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /build/aws-sdk-java-v2/app/src/main/java/io/minio/awssdk/v2/tests/MintLogger.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Mint, (C) 2018-2023 Minio, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.minio.awssdk.v2.tests; 19 | 20 | import com.fasterxml.jackson.annotation.JsonAutoDetect; 21 | import com.fasterxml.jackson.annotation.JsonIgnore; 22 | import com.fasterxml.jackson.annotation.JsonInclude.Include; 23 | import com.fasterxml.jackson.annotation.JsonProperty; 24 | import com.fasterxml.jackson.core.JsonProcessingException; 25 | import com.fasterxml.jackson.databind.ObjectMapper; 26 | 27 | @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) 28 | public class MintLogger { 29 | @JsonProperty("name") 30 | private String name; 31 | 32 | @JsonProperty("function") 33 | private String function; 34 | 35 | @JsonProperty("args") 36 | private String args; 37 | 38 | @JsonProperty("duration") 39 | private long duration; 40 | 41 | @JsonProperty("status") 42 | private String status; 43 | 44 | @JsonProperty("alert") 45 | private String alert; 46 | 47 | @JsonProperty("message") 48 | private String message; 49 | 50 | @JsonProperty("error") 51 | private String error; 52 | 53 | /** 54 | * Constructor. 55 | **/ 56 | public MintLogger(String function, 57 | String args, 58 | long duration, 59 | String status, 60 | String alert, 61 | String message, 62 | String error) { 63 | this.name = "aws-sdk-java-v2"; 64 | this.function = function; 65 | this.duration = duration; 66 | this.args = args; 67 | this.status = status; 68 | this.alert = alert; 69 | this.message = message; 70 | this.error = error; 71 | } 72 | 73 | /** 74 | * Return JSON Log Entry. 75 | **/ 76 | @JsonIgnore 77 | public String toString() { 78 | 79 | try { 80 | return new ObjectMapper().setSerializationInclusion(Include.NON_NULL).writeValueAsString(this); 81 | } catch (JsonProcessingException e) { 82 | e.printStackTrace(); 83 | } 84 | return ""; 85 | } 86 | 87 | /** 88 | * Return Alert. 89 | **/ 90 | @JsonIgnore 91 | public String alert() { 92 | return alert; 93 | } 94 | 95 | /** 96 | * Return Error. 97 | **/ 98 | @JsonIgnore 99 | public String error() { 100 | return error; 101 | } 102 | 103 | /** 104 | * Return Message. 105 | **/ 106 | @JsonIgnore 107 | public String message() { 108 | return message; 109 | } 110 | 111 | /** 112 | * Return args. 113 | **/ 114 | @JsonIgnore 115 | public String args() { 116 | return args; 117 | } 118 | 119 | /** 120 | * Return status. 121 | **/ 122 | @JsonIgnore 123 | public String status() { 124 | return status; 125 | } 126 | 127 | /** 128 | * Return name. 129 | **/ 130 | @JsonIgnore 131 | public String name() { 132 | return name; 133 | } 134 | 135 | /** 136 | * Return function. 137 | **/ 138 | @JsonIgnore 139 | public String function() { 140 | return function; 141 | } 142 | 143 | /** 144 | * Return duration. 145 | **/ 146 | @JsonIgnore 147 | public long duration() { 148 | return duration; 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /build/aws-sdk-java-v2/app/src/main/java/io/minio/awssdk/v2/tests/S3TestUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Mint, (C) 2018-2023 Minio, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.minio.awssdk.v2.tests; 19 | 20 | import software.amazon.awssdk.core.ResponseInputStream; 21 | import software.amazon.awssdk.core.async.AsyncRequestBody; 22 | import software.amazon.awssdk.core.sync.RequestBody; 23 | import software.amazon.awssdk.services.s3.S3AsyncClient; 24 | import software.amazon.awssdk.services.s3.S3Client; 25 | import software.amazon.awssdk.services.s3.model.*; 26 | 27 | 28 | import java.io.File; 29 | import java.io.IOException; 30 | import java.nio.ByteBuffer; 31 | import java.util.ArrayList; 32 | import java.util.List; 33 | import java.util.Random; 34 | import java.util.concurrent.CompletableFuture; 35 | 36 | public class S3TestUtils { 37 | private final S3Client s3Client; 38 | private final S3AsyncClient s3AsyncClient; 39 | private final S3AsyncClient s3CrtAsyncClient; 40 | 41 | S3TestUtils(S3Client s3Client, S3AsyncClient s3AsyncClient, S3AsyncClient s3CrtAsyncClient) { 42 | this.s3Client = s3Client; 43 | this.s3AsyncClient = s3AsyncClient; 44 | this.s3CrtAsyncClient = s3CrtAsyncClient; 45 | } 46 | 47 | void uploadMultipartObject(String bucketName, String keyName) throws IOException { 48 | int mB = 1024 * 1024; 49 | 50 | CreateMultipartUploadRequest.Builder builder = CreateMultipartUploadRequest.builder(); 51 | builder.bucket(bucketName).key(keyName); 52 | 53 | // Step 1: Initialize. 54 | CreateMultipartUploadResponse res = s3Client.createMultipartUpload(builder.build()); 55 | String uploadId = res.uploadId(); 56 | List completedParts = new ArrayList<>(); 57 | 58 | // Step 2: Upload parts. 59 | for (int i = 1; i <= 10; i++) { 60 | UploadPartRequest req = UploadPartRequest 61 | .builder() 62 | .bucket(bucketName) 63 | .key(keyName) 64 | .uploadId(uploadId) 65 | .partNumber(i) 66 | .build(); 67 | UploadPartResponse resp = s3Client.uploadPart(req, RequestBody.fromByteBuffer(getRandomByteBuffer(5 * mB))); 68 | String etag = resp.eTag(); 69 | CompletedPart part = CompletedPart 70 | .builder() 71 | .partNumber(i) 72 | .eTag(etag) 73 | .build(); 74 | completedParts.add(part); 75 | } 76 | 77 | // Step 3: Complete. 78 | CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder().parts(completedParts).build(); 79 | CompleteMultipartUploadRequest completeMultipartUploadRequest = 80 | CompleteMultipartUploadRequest.builder() 81 | .bucket(bucketName).key(keyName) 82 | .uploadId(uploadId) 83 | .multipartUpload(completedMultipartUpload) 84 | .build(); 85 | s3Client.completeMultipartUpload(completeMultipartUploadRequest); 86 | } 87 | 88 | void uploadMultipartObjectAsync(String bucketName, String keyName) throws Exception { 89 | int mB = 1024 * 1024; 90 | 91 | CreateMultipartUploadRequest.Builder builder = CreateMultipartUploadRequest.builder(); 92 | builder.bucket(bucketName).key(keyName); 93 | 94 | // Step 1: Initialize. 95 | CompletableFuture res = s3AsyncClient.createMultipartUpload(builder.build()); 96 | String uploadId = res.get().uploadId(); 97 | List completedParts = new ArrayList<>(); 98 | 99 | // Step 2: Upload parts. 100 | for (int i = 1; i <= 10; i++) { 101 | UploadPartRequest req = UploadPartRequest 102 | .builder() 103 | .bucket(bucketName) 104 | .key(keyName) 105 | .uploadId(uploadId) 106 | .partNumber(i) 107 | .build(); 108 | CompletableFuture resp = s3AsyncClient.uploadPart(req, AsyncRequestBody.fromByteBuffer(getRandomByteBuffer(5 * mB))); 109 | String etag = resp.get().eTag(); 110 | CompletedPart part = CompletedPart 111 | .builder() 112 | .partNumber(i) 113 | .eTag(etag) 114 | .build(); 115 | completedParts.add(part); 116 | } 117 | 118 | // Step 3: Complete. 119 | CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder().parts(completedParts).build(); 120 | CompleteMultipartUploadRequest completeMultipartUploadRequest = 121 | CompleteMultipartUploadRequest.builder() 122 | .bucket(bucketName).key(keyName) 123 | .uploadId(uploadId) 124 | .multipartUpload(completedMultipartUpload) 125 | .build(); 126 | s3AsyncClient.completeMultipartUpload(completeMultipartUploadRequest); 127 | } 128 | 129 | private static ByteBuffer getRandomByteBuffer(int size) throws IOException { 130 | byte[] b = new byte[size]; 131 | new Random().nextBytes(b); 132 | return ByteBuffer.wrap(b); 133 | } 134 | 135 | void uploadObject(String bucketName, String keyName, String filePath) throws IOException { 136 | 137 | File f = new File(filePath); 138 | PutObjectRequest request = PutObjectRequest 139 | .builder() 140 | .bucket(bucketName) 141 | .key(keyName) 142 | .build(); 143 | s3Client.putObject(request, RequestBody.fromFile(f)); 144 | } 145 | 146 | void downloadObject(String bucketName, String keyName, String expectedMD5) throws Exception, IOException { 147 | GetObjectRequest request = GetObjectRequest 148 | .builder() 149 | .bucket(bucketName) 150 | .key(keyName) 151 | .build(); 152 | ResponseInputStream response = s3Client.getObject(request); 153 | 154 | String calculatedMD5 = Utils.getBufferMD5(response.readAllBytes()); 155 | 156 | if (!expectedMD5.equals("") && !calculatedMD5.equals(expectedMD5)) { 157 | throw new Exception("downloaded object has unexpected md5sum, expected: " + expectedMD5 + ", found: " + calculatedMD5); 158 | } 159 | } 160 | 161 | void copyObject(String bucketName, String keyName, 162 | String targetBucketName, String targetKeyName, String newSseKey, 163 | boolean replace) { 164 | CopyObjectRequest.Builder builder = CopyObjectRequest.builder(); 165 | builder.sourceBucket(bucketName).sourceKey(keyName).destinationBucket(targetBucketName).destinationKey(targetKeyName); 166 | if (replace) { 167 | builder.metadataDirective(MetadataDirective.COPY); 168 | } 169 | s3Client.copyObject(builder.build()); 170 | } 171 | 172 | long retrieveObjectMetadata(String bucketName, String keyName, String sseKey) { 173 | GetObjectRequest request = GetObjectRequest 174 | .builder() 175 | .bucket(bucketName) 176 | .key(keyName) 177 | .sseCustomerKey(sseKey) 178 | .build(); 179 | ResponseInputStream response = s3Client.getObject(request); 180 | return response.response().contentLength(); 181 | } 182 | } 183 | -------------------------------------------------------------------------------- /build/aws-sdk-java-v2/app/src/main/java/io/minio/awssdk/v2/tests/Utils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Mint, (C) 2018-2023 Minio, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.minio.awssdk.v2.tests; 19 | 20 | import java.io.ByteArrayInputStream; 21 | import java.io.File; 22 | import java.io.FileInputStream; 23 | import java.io.InputStream; 24 | import java.security.MessageDigest; 25 | 26 | public class Utils { 27 | public static byte[] createChecksum(InputStream is, int skip, int length) throws Exception { 28 | int numRead; 29 | byte[] buffer = new byte[1024]; 30 | 31 | MessageDigest complete = MessageDigest.getInstance("MD5"); 32 | 33 | if (skip > -1 && length > -1) { 34 | is = new LimitedInputStream(is, skip, length); 35 | } 36 | 37 | do { 38 | numRead = is.read(buffer); 39 | if (numRead > 0) { 40 | complete.update(buffer, 0, numRead); 41 | } 42 | } while (numRead != -1); 43 | 44 | return complete.digest(); 45 | } 46 | 47 | public static String getInputStreamMD5(InputStream is) throws Exception { 48 | return getInputStreamMD5(is, -1, -1); 49 | } 50 | 51 | public static String getInputStreamMD5(InputStream is, int start, int length) throws Exception { 52 | byte[] b = createChecksum(is, start, length); 53 | String result = ""; 54 | 55 | for (int i=0; i < b.length; i++) { 56 | result += Integer.toString( ( b[i] & 0xff ) + 0x100, 16).substring( 1 ); 57 | } 58 | return result; 59 | } 60 | 61 | public static String getFileMD5(String filePath) throws Exception { 62 | return getFileMD5(filePath, -1, -1); 63 | } 64 | 65 | public static String getFileMD5(String filePath, int start, int length) throws Exception { 66 | File f = new File(filePath); 67 | InputStream is = new FileInputStream(f); 68 | return getInputStreamMD5(is, start, length); 69 | } 70 | 71 | public static String getBufferMD5(byte[] data) throws Exception { 72 | ByteArrayInputStream bis = new ByteArrayInputStream(data); 73 | return getInputStreamMD5(bis); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /build/aws-sdk-java-v2/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip 4 | networkTimeout=10000 5 | validateDistributionUrl=true 6 | zipStoreBase=GRADLE_USER_HOME 7 | zipStorePath=wrapper/dists 8 | -------------------------------------------------------------------------------- /build/aws-sdk-java-v2/gradlew: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Copyright © 2015-2021 the original authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | # 21 | # Gradle start up script for POSIX generated by Gradle. 22 | # 23 | # Important for running: 24 | # 25 | # (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is 26 | # noncompliant, but you have some other compliant shell such as ksh or 27 | # bash, then to run this script, type that shell name before the whole 28 | # command line, like: 29 | # 30 | # ksh Gradle 31 | # 32 | # Busybox and similar reduced shells will NOT work, because this script 33 | # requires all of these POSIX shell features: 34 | # * functions; 35 | # * expansions «$var», «${var}», «${var:-default}», «${var+SET}», 36 | # «${var#prefix}», «${var%suffix}», and «$( cmd )»; 37 | # * compound commands having a testable exit status, especially «case»; 38 | # * various built-in commands including «command», «set», and «ulimit». 39 | # 40 | # Important for patching: 41 | # 42 | # (2) This script targets any POSIX shell, so it avoids extensions provided 43 | # by Bash, Ksh, etc; in particular arrays are avoided. 44 | # 45 | # The "traditional" practice of packing multiple parameters into a 46 | # space-separated string is a well documented source of bugs and security 47 | # problems, so this is (mostly) avoided, by progressively accumulating 48 | # options in "$@", and eventually passing that to Java. 49 | # 50 | # Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, 51 | # and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; 52 | # see the in-line comments for details. 53 | # 54 | # There are tweaks for specific operating systems such as AIX, CygWin, 55 | # Darwin, MinGW, and NonStop. 56 | # 57 | # (3) This script is generated from the Groovy template 58 | # https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt 59 | # within the Gradle project. 60 | # 61 | # You can find Gradle at https://github.com/gradle/gradle/. 62 | # 63 | ############################################################################## 64 | 65 | # Attempt to set APP_HOME 66 | 67 | # Resolve links: $0 may be a link 68 | app_path=$0 69 | 70 | # Need this for daisy-chained symlinks. 71 | while 72 | APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path 73 | [ -h "$app_path" ] 74 | do 75 | ls=$( ls -ld "$app_path" ) 76 | link=${ls#*' -> '} 77 | case $link in #( 78 | /*) app_path=$link ;; #( 79 | *) app_path=$APP_HOME$link ;; 80 | esac 81 | done 82 | 83 | # This is normally unused 84 | # shellcheck disable=SC2034 85 | APP_BASE_NAME=${0##*/} 86 | # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) 87 | APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit 88 | 89 | # Use the maximum available, or set MAX_FD != -1 to use that value. 90 | MAX_FD=maximum 91 | 92 | warn () { 93 | echo "$*" 94 | } >&2 95 | 96 | die () { 97 | echo 98 | echo "$*" 99 | echo 100 | exit 1 101 | } >&2 102 | 103 | # OS specific support (must be 'true' or 'false'). 104 | cygwin=false 105 | msys=false 106 | darwin=false 107 | nonstop=false 108 | case "$( uname )" in #( 109 | CYGWIN* ) cygwin=true ;; #( 110 | Darwin* ) darwin=true ;; #( 111 | MSYS* | MINGW* ) msys=true ;; #( 112 | NONSTOP* ) nonstop=true ;; 113 | esac 114 | 115 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 116 | 117 | 118 | # Determine the Java command to use to start the JVM. 119 | if [ -n "$JAVA_HOME" ] ; then 120 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 121 | # IBM's JDK on AIX uses strange locations for the executables 122 | JAVACMD=$JAVA_HOME/jre/sh/java 123 | else 124 | JAVACMD=$JAVA_HOME/bin/java 125 | fi 126 | if [ ! -x "$JAVACMD" ] ; then 127 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 128 | 129 | Please set the JAVA_HOME variable in your environment to match the 130 | location of your Java installation." 131 | fi 132 | else 133 | JAVACMD=java 134 | if ! command -v java >/dev/null 2>&1 135 | then 136 | die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 137 | 138 | Please set the JAVA_HOME variable in your environment to match the 139 | location of your Java installation." 140 | fi 141 | fi 142 | 143 | # Increase the maximum file descriptors if we can. 144 | if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then 145 | case $MAX_FD in #( 146 | max*) 147 | # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. 148 | # shellcheck disable=SC2039,SC3045 149 | MAX_FD=$( ulimit -H -n ) || 150 | warn "Could not query maximum file descriptor limit" 151 | esac 152 | case $MAX_FD in #( 153 | '' | soft) :;; #( 154 | *) 155 | # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. 156 | # shellcheck disable=SC2039,SC3045 157 | ulimit -n "$MAX_FD" || 158 | warn "Could not set maximum file descriptor limit to $MAX_FD" 159 | esac 160 | fi 161 | 162 | # Collect all arguments for the java command, stacking in reverse order: 163 | # * args from the command line 164 | # * the main class name 165 | # * -classpath 166 | # * -D...appname settings 167 | # * --module-path (only if needed) 168 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. 169 | 170 | # For Cygwin or MSYS, switch paths to Windows format before running java 171 | if "$cygwin" || "$msys" ; then 172 | APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) 173 | CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) 174 | 175 | JAVACMD=$( cygpath --unix "$JAVACMD" ) 176 | 177 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 178 | for arg do 179 | if 180 | case $arg in #( 181 | -*) false ;; # don't mess with options #( 182 | /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath 183 | [ -e "$t" ] ;; #( 184 | *) false ;; 185 | esac 186 | then 187 | arg=$( cygpath --path --ignore --mixed "$arg" ) 188 | fi 189 | # Roll the args list around exactly as many times as the number of 190 | # args, so each arg winds up back in the position where it started, but 191 | # possibly modified. 192 | # 193 | # NB: a `for` loop captures its iteration list before it begins, so 194 | # changing the positional parameters here affects neither the number of 195 | # iterations, nor the values presented in `arg`. 196 | shift # remove old arg 197 | set -- "$@" "$arg" # push replacement arg 198 | done 199 | fi 200 | 201 | 202 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 203 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 204 | 205 | # Collect all arguments for the java command: 206 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, 207 | # and any embedded shellness will be escaped. 208 | # * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be 209 | # treated as '${Hostname}' itself on the command line. 210 | 211 | set -- \ 212 | "-Dorg.gradle.appname=$APP_BASE_NAME" \ 213 | -classpath "$CLASSPATH" \ 214 | org.gradle.wrapper.GradleWrapperMain \ 215 | "$@" 216 | 217 | # Stop when "xargs" is not available. 218 | if ! command -v xargs >/dev/null 2>&1 219 | then 220 | die "xargs is not available" 221 | fi 222 | 223 | # Use "xargs" to parse quoted args. 224 | # 225 | # With -n1 it outputs one arg per line, with the quotes and backslashes removed. 226 | # 227 | # In Bash we could simply go: 228 | # 229 | # readarray ARGS < <( xargs -n1 <<<"$var" ) && 230 | # set -- "${ARGS[@]}" "$@" 231 | # 232 | # but POSIX shell has neither arrays nor command substitution, so instead we 233 | # post-process each arg (as a line of input to sed) to backslash-escape any 234 | # character that might be a shell metacharacter, then use eval to reverse 235 | # that process (while maintaining the separation between arguments), and wrap 236 | # the whole thing up as a single "set" statement. 237 | # 238 | # This will of course break if any of these variables contains a newline or 239 | # an unmatched quote. 240 | # 241 | 242 | eval "set -- $( 243 | printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | 244 | xargs -n1 | 245 | sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | 246 | tr '\n' ' ' 247 | )" '"$@"' 248 | 249 | exec "$JAVACMD" "$@" 250 | -------------------------------------------------------------------------------- /build/aws-sdk-java-v2/gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%"=="" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%"=="" set DIRNAME=. 29 | @rem This is normally unused 30 | set APP_BASE_NAME=%~n0 31 | set APP_HOME=%DIRNAME% 32 | 33 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 34 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 35 | 36 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 37 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 38 | 39 | @rem Find java.exe 40 | if defined JAVA_HOME goto findJavaFromJavaHome 41 | 42 | set JAVA_EXE=java.exe 43 | %JAVA_EXE% -version >NUL 2>&1 44 | if %ERRORLEVEL% equ 0 goto execute 45 | 46 | echo. 47 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 48 | echo. 49 | echo Please set the JAVA_HOME variable in your environment to match the 50 | echo location of your Java installation. 51 | 52 | goto fail 53 | 54 | :findJavaFromJavaHome 55 | set JAVA_HOME=%JAVA_HOME:"=% 56 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 57 | 58 | if exist "%JAVA_EXE%" goto execute 59 | 60 | echo. 61 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 62 | echo. 63 | echo Please set the JAVA_HOME variable in your environment to match the 64 | echo location of your Java installation. 65 | 66 | goto fail 67 | 68 | :execute 69 | @rem Setup the command line 70 | 71 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 72 | 73 | 74 | @rem Execute Gradle 75 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 76 | 77 | :end 78 | @rem End local scope for the variables with windows NT shell 79 | if %ERRORLEVEL% equ 0 goto mainEnd 80 | 81 | :fail 82 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 83 | rem the _cmd.exe /c_ return code! 84 | set EXIT_CODE=%ERRORLEVEL% 85 | if %EXIT_CODE% equ 0 set EXIT_CODE=1 86 | if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% 87 | exit /b %EXIT_CODE% 88 | 89 | :mainEnd 90 | if "%OS%"=="Windows_NT" endlocal 91 | 92 | :omega 93 | -------------------------------------------------------------------------------- /build/aws-sdk-java-v2/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | test_run_dir="$MINT_RUN_CORE_DIR/aws-sdk-java-v2" 19 | 20 | cd "$(dirname "$(realpath "$0")")" 21 | 22 | gradle wrapper 23 | ./gradlew clean build 24 | 25 | cp app/build/libs/FunctionalTests.jar "$test_run_dir/" 26 | 27 | rm -rf app/build/ 28 | -------------------------------------------------------------------------------- /build/aws-sdk-java-v2/settings.gradle.kts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file was generated by the Gradle 'init' task. 3 | * 4 | * The settings file is used to specify which projects to include in your build. 5 | * For more detailed information on multi-project builds, please refer to https://docs.gradle.org/8.5/userguide/building_swift_projects.html in the Gradle documentation. 6 | */ 7 | 8 | plugins { 9 | // Apply the foojay-resolver plugin to allow automatic download of JDKs 10 | id("org.gradle.toolchains.foojay-resolver-convention") version "0.7.0" 11 | } 12 | 13 | rootProject.name = "aws-sdk-java-v2" 14 | include("app") 15 | -------------------------------------------------------------------------------- /build/aws-sdk-java/build.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /build/aws-sdk-java/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | test_run_dir="$MINT_RUN_CORE_DIR/aws-sdk-java" 19 | 20 | cd "$(dirname "$(realpath "$0")")" 21 | 22 | ant init-ivy && 23 | ant resolve && 24 | ant compile && 25 | ant jar 26 | 27 | cp build/jar/FunctionalTests.jar "$test_run_dir/" 28 | 29 | rm -rf lib/ build/ 30 | -------------------------------------------------------------------------------- /build/aws-sdk-java/ivy.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /build/aws-sdk-java/src/LimitedInputStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Mint, (C) 2018 Minio, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.minio.awssdk.tests; 19 | 20 | import java.io.*; 21 | 22 | // LimitedInputStream wraps a regular InputStream, calling 23 | // read() will skip some bytes as configured and will also 24 | // return only data with configured length 25 | 26 | class LimitedInputStream extends InputStream { 27 | 28 | private int skip; 29 | private int length; 30 | private InputStream is; 31 | 32 | LimitedInputStream(InputStream is, int skip, int length) { 33 | this.is = is; 34 | this.skip = skip; 35 | this.length = length; 36 | } 37 | 38 | @Override 39 | public int read() throws IOException { 40 | int r; 41 | while (skip > 0) { 42 | r = is.read(); 43 | if (r < 0) { 44 | throw new IOException("stream ended before being able to skip all bytes"); 45 | } 46 | skip--; 47 | } 48 | if (length == 0) { 49 | return -1; 50 | } 51 | r = is.read(); 52 | if (r < 0) { 53 | throw new IOException("stream ended before being able to read all bytes"); 54 | } 55 | length--; 56 | return r; 57 | } 58 | } 59 | 60 | 61 | -------------------------------------------------------------------------------- /build/aws-sdk-java/src/MintLogger.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Java SDK for Amazon S3 Compatible Cloud Storage, 3 | * (C) 2015, 2016, 2017, 2018 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.minio.awssdk.tests; 19 | 20 | import com.fasterxml.jackson.annotation.JsonAutoDetect; 21 | import com.fasterxml.jackson.annotation.JsonIgnore; 22 | import com.fasterxml.jackson.annotation.JsonInclude.Include; 23 | import com.fasterxml.jackson.annotation.JsonProperty; 24 | import com.fasterxml.jackson.core.JsonProcessingException; 25 | import com.fasterxml.jackson.databind.ObjectMapper; 26 | 27 | @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) 28 | public class MintLogger { 29 | 30 | @JsonProperty("name") 31 | private String name; 32 | 33 | @JsonProperty("function") 34 | private String function; 35 | 36 | @JsonProperty("args") 37 | private String args; 38 | 39 | @JsonProperty("duration") 40 | private long duration; 41 | 42 | @JsonProperty("status") 43 | private String status; 44 | 45 | @JsonProperty("alert") 46 | private String alert; 47 | 48 | @JsonProperty("message") 49 | private String message; 50 | 51 | @JsonProperty("error") 52 | private String error; 53 | 54 | /** 55 | * Constructor. 56 | **/ 57 | public MintLogger(String function, 58 | String args, 59 | long duration, 60 | String status, 61 | String alert, 62 | String message, 63 | String error) { 64 | this.name = "aws-sdk-java"; 65 | this.function = function; 66 | this.duration = duration; 67 | this.args = args; 68 | this.status = status; 69 | this.alert = alert; 70 | this.message = message; 71 | this.error = error; 72 | } 73 | 74 | /** 75 | * Return JSON Log Entry. 76 | **/ 77 | @JsonIgnore 78 | public String toString() { 79 | 80 | try { 81 | return new ObjectMapper().setSerializationInclusion(Include.NON_NULL).writeValueAsString(this); 82 | } catch (JsonProcessingException e) { 83 | e.printStackTrace(); 84 | } 85 | return ""; 86 | } 87 | 88 | /** 89 | * Return Alert. 90 | **/ 91 | @JsonIgnore 92 | public String alert() { 93 | return alert; 94 | } 95 | 96 | /** 97 | * Return Error. 98 | **/ 99 | @JsonIgnore 100 | public String error() { 101 | return error; 102 | } 103 | 104 | /** 105 | * Return Message. 106 | **/ 107 | @JsonIgnore 108 | public String message() { 109 | return message; 110 | } 111 | 112 | /** 113 | * Return args. 114 | **/ 115 | @JsonIgnore 116 | public String args() { 117 | return args; 118 | } 119 | 120 | /** 121 | * Return status. 122 | **/ 123 | @JsonIgnore 124 | public String status() { 125 | return status; 126 | } 127 | 128 | /** 129 | * Return name. 130 | **/ 131 | @JsonIgnore 132 | public String name() { 133 | return name; 134 | } 135 | 136 | /** 137 | * Return function. 138 | **/ 139 | @JsonIgnore 140 | public String function() { 141 | return function; 142 | } 143 | 144 | /** 145 | * Return duration. 146 | **/ 147 | @JsonIgnore 148 | public long duration() { 149 | return duration; 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /build/aws-sdk-java/src/S3TestUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Mint, (C) 2018 Minio, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package io.minio.awssdk.tests; 19 | 20 | import java.io.*; 21 | import java.util.*; 22 | import java.nio.channels.Channels; 23 | 24 | import com.amazonaws.services.s3.model.GetObjectMetadataRequest; 25 | import com.amazonaws.services.s3.model.GetObjectRequest; 26 | import com.amazonaws.services.s3.model.ObjectMetadata; 27 | import com.amazonaws.services.s3.model.PutObjectRequest; 28 | import com.amazonaws.services.s3.model.CopyObjectRequest; 29 | import com.amazonaws.services.s3.model.S3Object; 30 | import com.amazonaws.services.s3.model.S3ObjectInputStream; 31 | import com.amazonaws.services.s3.model.SSECustomerKey; 32 | 33 | import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; 34 | import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; 35 | import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; 36 | import com.amazonaws.services.s3.model.PartETag; 37 | import com.amazonaws.services.s3.model.UploadPartRequest; 38 | 39 | import com.amazonaws.services.s3.model.MetadataDirective; 40 | 41 | import com.amazonaws.services.s3.AmazonS3; 42 | 43 | class S3TestUtils { 44 | 45 | private AmazonS3 s3Client; 46 | 47 | S3TestUtils(AmazonS3 s3Client) { 48 | this.s3Client = s3Client; 49 | } 50 | 51 | void uploadMultipartObject(String bucketName, String keyName, 52 | String filePath, SSECustomerKey sseKey) throws IOException { 53 | 54 | File file = new File(filePath); 55 | 56 | List partETags = new ArrayList(); 57 | 58 | // Step 1: Initialize. 59 | InitiateMultipartUploadRequest initRequest = new 60 | InitiateMultipartUploadRequest(bucketName, keyName); 61 | 62 | if (sseKey != null) { 63 | initRequest.setSSECustomerKey(sseKey); 64 | } 65 | 66 | InitiateMultipartUploadResult initResponse = 67 | s3Client.initiateMultipartUpload(initRequest); 68 | 69 | long contentLength = file.length(); 70 | long partSize = 5242880; // Set part size to 5 MB. 71 | 72 | // Step 2: Upload parts. 73 | long filePosition = 0; 74 | for (int i = 1; filePosition < contentLength; i++) { 75 | // Last part can be less than 5 MB. Adjust part size. 76 | partSize = Math.min(partSize, (contentLength - filePosition)); 77 | 78 | // Create request to upload a part. 79 | UploadPartRequest uploadRequest = new UploadPartRequest() 80 | .withBucketName(bucketName).withKey(keyName) 81 | .withUploadId(initResponse.getUploadId()).withPartNumber(i) 82 | .withFileOffset(filePosition) 83 | .withFile(file) 84 | .withPartSize(partSize); 85 | 86 | if (sseKey != null) { 87 | uploadRequest.withSSECustomerKey(sseKey); 88 | } 89 | 90 | // Upload part and add response to our list. 91 | partETags.add(s3Client.uploadPart(uploadRequest).getPartETag()); 92 | 93 | filePosition += partSize; 94 | } 95 | 96 | // Step 3: Complete. 97 | CompleteMultipartUploadRequest compRequest = new 98 | CompleteMultipartUploadRequest( 99 | bucketName, 100 | keyName, 101 | initResponse.getUploadId(), 102 | partETags); 103 | 104 | s3Client.completeMultipartUpload(compRequest); 105 | } 106 | 107 | void uploadObject(String bucketName, String keyName, 108 | String filePath, SSECustomerKey sseKey) throws IOException { 109 | 110 | File f = new File(filePath); 111 | PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, keyName, f); 112 | if (sseKey != null) { 113 | putObjectRequest.withSSECustomerKey(sseKey); 114 | } 115 | s3Client.putObject(putObjectRequest); 116 | } 117 | 118 | void downloadObject(String bucketName, String keyName, SSECustomerKey sseKey) 119 | throws Exception, IOException { 120 | downloadObject(bucketName, keyName, sseKey, "", -1, -1); 121 | } 122 | 123 | void downloadObject(String bucketName, String keyName, SSECustomerKey sseKey, 124 | String expectedMD5) 125 | throws Exception, IOException { 126 | downloadObject(bucketName, keyName, sseKey, expectedMD5, -1, -1); 127 | } 128 | 129 | void downloadObject(String bucketName, String keyName, SSECustomerKey sseKey, 130 | String expectedMD5, int start, int length) throws Exception, IOException { 131 | GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, keyName) 132 | .withSSECustomerKey(sseKey); 133 | 134 | if (start >= 0 && length >= 0) { 135 | getObjectRequest.setRange(start, start+length-1); 136 | } 137 | 138 | S3Object s3Object = s3Client.getObject(getObjectRequest); 139 | 140 | int size = 0; 141 | int c; 142 | 143 | S3ObjectInputStream input = s3Object.getObjectContent(); 144 | 145 | ByteArrayOutputStream output = new ByteArrayOutputStream(); 146 | String data = ""; 147 | while ((c = input.read()) != -1) { 148 | output.write((byte) c); 149 | size++; 150 | } 151 | 152 | if (length >= 0 && size != length) { 153 | throw new Exception("downloaded object has unexpected size, expected: " + length + ", received: " + size); 154 | } 155 | 156 | String calculatedMD5 = Utils.getBufferMD5(output.toByteArray()); 157 | 158 | if (!expectedMD5.equals("") && !calculatedMD5.equals(expectedMD5)) { 159 | throw new Exception("downloaded object has unexpected md5sum, expected: " + expectedMD5 + ", found: " + calculatedMD5); 160 | 161 | } 162 | } 163 | 164 | void copyObject(String bucketName, String keyName, SSECustomerKey sseKey, 165 | String targetBucketName, String targetKeyName, SSECustomerKey newSseKey, 166 | boolean replace) { 167 | CopyObjectRequest copyRequest = new CopyObjectRequest(bucketName, keyName, targetBucketName, targetKeyName); 168 | if (sseKey != null) { 169 | copyRequest.withSourceSSECustomerKey(sseKey); 170 | } 171 | if (newSseKey != null) { 172 | copyRequest.withDestinationSSECustomerKey(newSseKey); 173 | } 174 | if (replace) { 175 | copyRequest.withMetadataDirective(MetadataDirective.COPY); 176 | } 177 | s3Client.copyObject(copyRequest); 178 | } 179 | 180 | long retrieveObjectMetadata(String bucketName, String keyName, SSECustomerKey sseKey) { 181 | GetObjectMetadataRequest getMetadataRequest = new GetObjectMetadataRequest(bucketName, keyName) 182 | .withSSECustomerKey(sseKey); 183 | ObjectMetadata objectMetadata = s3Client.getObjectMetadata(getMetadataRequest); 184 | return objectMetadata.getContentLength(); 185 | } 186 | 187 | } 188 | -------------------------------------------------------------------------------- /build/aws-sdk-java/src/Utils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Mint, (C) 2018 Minio, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | 19 | package io.minio.awssdk.tests; 20 | 21 | import java.io.*; 22 | import java.nio.channels.*; 23 | import java.security.*; 24 | 25 | class Utils { 26 | 27 | public static byte[] createChecksum(InputStream is, int skip, int length) throws Exception { 28 | int numRead; 29 | byte[] buffer = new byte[1024]; 30 | 31 | MessageDigest complete = MessageDigest.getInstance("MD5"); 32 | 33 | if (skip > -1 && length > -1) { 34 | is = new LimitedInputStream(is, skip, length); 35 | } 36 | 37 | do { 38 | numRead = is.read(buffer); 39 | if (numRead > 0) { 40 | complete.update(buffer, 0, numRead); 41 | } 42 | } while (numRead != -1); 43 | 44 | return complete.digest(); 45 | } 46 | 47 | public static String getInputStreamMD5(InputStream is) throws Exception { 48 | return getInputStreamMD5(is, -1, -1); 49 | } 50 | 51 | public static String getInputStreamMD5(InputStream is, int start, int length) throws Exception { 52 | byte[] b = createChecksum(is, start, length); 53 | String result = ""; 54 | 55 | for (int i=0; i < b.length; i++) { 56 | result += Integer.toString( ( b[i] & 0xff ) + 0x100, 16).substring( 1 ); 57 | } 58 | return result; 59 | } 60 | 61 | public static String getFileMD5(String filePath) throws Exception { 62 | return getFileMD5(filePath, -1, -1); 63 | } 64 | 65 | public static String getFileMD5(String filePath, int start, int length) throws Exception { 66 | File f = new File(filePath); 67 | InputStream is = new FileInputStream(f); 68 | return getInputStreamMD5(is, start, length); 69 | } 70 | 71 | public static String getBufferMD5(byte[] data) throws Exception { 72 | ByteArrayInputStream bis = new ByteArrayInputStream(data); 73 | return getInputStreamMD5(bis); 74 | } 75 | } 76 | 77 | -------------------------------------------------------------------------------- /build/aws-sdk-php/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | test_run_dir="$MINT_RUN_CORE_DIR/aws-sdk-php" 19 | $WGET --output-document=- https://getcomposer.org/installer | php -- --install-dir="$test_run_dir" 20 | php "$test_run_dir/composer.phar" --working-dir="$test_run_dir" install 21 | -------------------------------------------------------------------------------- /build/aws-sdk-ruby/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Mint (C) 20172-2020 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | gem install --no-document aws-sdk-resources aws-sdk multipart_body 19 | -------------------------------------------------------------------------------- /build/awscli/expect-100.patch: -------------------------------------------------------------------------------- 1 | diff --git a/botocore/handlers.py b/botocore/handlers.py 2 | index 4b1d6445e..c8dd79c3e 100644 3 | --- a/botocore/handlers.py 4 | +++ b/botocore/handlers.py 5 | @@ -333,9 +333,11 @@ def add_expect_header(model, params, **kwargs): 6 | return 7 | if 'body' in params: 8 | body = params['body'] 9 | - if hasattr(body, 'read'): 10 | + size = getattr(body, "_size", None) 11 | + if size: 12 | # Any file like object will use an expect 100-continue 13 | - # header regardless of size. 14 | + # except when size equals zero. 15 | + # https://tools.ietf.org/html/rfc7231#section-5.1.1 16 | logger.debug("Adding expect 100 continue header to request.") 17 | params['headers']['Expect'] = '100-continue' 18 | 19 | diff --git a/tests/unit/test_awsrequest.py b/tests/unit/test_awsrequest.py 20 | index 22bd9a746..44cd05f61 100644 21 | --- a/tests/unit/test_awsrequest.py 22 | +++ b/tests/unit/test_awsrequest.py 23 | @@ -504,6 +504,14 @@ class TestAWSHTTPConnection(unittest.TestCase): 24 | response = conn.getresponse() 25 | self.assertEqual(response.status, 200) 26 | 27 | + def test_no_expect_header_set_no_body(self): 28 | + s = FakeSocket(b'HTTP/1.1 200 OK\r\n') 29 | + conn = AWSHTTPConnection('s3.amazonaws.com', 443) 30 | + conn.sock = s 31 | + conn.request('PUT', '/bucket/foo', b'') 32 | + response = conn.getresponse() 33 | + self.assertEqual(response.status, 200) 34 | + 35 | def test_tunnel_readline_none_bugfix(self): 36 | # Tests whether ``_tunnel`` function is able to work around the 37 | # py26 bug of avoiding infinite while loop if nothing is returned. 38 | -------------------------------------------------------------------------------- /build/awscli/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Mint (C) 2017-2022 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | die() { 19 | echo "$*" 1>&2 20 | exit 1 21 | } 22 | 23 | # shellcheck disable=SC2086 24 | ROOTDIR="$(dirname "$(realpath $0)")" 25 | TMPDIR="$(mktemp -d)" 26 | 27 | cd "$TMPDIR" 28 | 29 | # Download botocore and apply @y4m4's expect 100 continue fix 30 | (git clone --depth 1 -b 1.31.37 https://github.com/boto/botocore && 31 | cd botocore && 32 | patch -p1 <"$ROOTDIR/expect-100.patch" && 33 | python3 -m pip install .) || 34 | die "Unable to install botocore.." 35 | 36 | # Download and install aws cli 37 | (git clone --depth 1 -b 1.29.37 https://github.com/aws/aws-cli && 38 | cd aws-cli && 39 | python3 -m pip install .) || 40 | die "Unable to install aws-cli.." 41 | 42 | # Clean-up 43 | rm -r "$TMPDIR" 44 | -------------------------------------------------------------------------------- /build/healthcheck/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Mint (C) 2019 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | test_run_dir="$MINT_RUN_CORE_DIR/healthcheck" 19 | (cd "$test_run_dir" && CGO_ENABLED=0 go build --ldflags "-s -w") 20 | -------------------------------------------------------------------------------- /build/mc/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | MC_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/mc/releases/latest | sed "s/https:\/\/github.com\/minio\/mc\/releases\/tag\///") 19 | if [ -z "$MC_VERSION" ]; then 20 | echo "unable to get mc version from github" 21 | exit 1 22 | fi 23 | 24 | test_run_dir="$MINT_RUN_CORE_DIR/mc" 25 | $WGET --output-document="${test_run_dir}/mc" "https://dl.minio.io/client/mc/release/linux-amd64/mc.${MC_VERSION}" 26 | chmod a+x "${test_run_dir}/mc" 27 | 28 | git clone --quiet https://github.com/minio/mc.git "$test_run_dir/mc.git" 29 | ( 30 | cd "$test_run_dir/mc.git" 31 | git checkout --quiet "tags/${MC_VERSION}" 32 | ) 33 | cp -a "${test_run_dir}/mc.git/functional-tests.sh" "$test_run_dir/" 34 | rm -fr "$test_run_dir/mc.git" 35 | -------------------------------------------------------------------------------- /build/minio-go/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | MINIO_GO_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/minio-go/releases/latest | sed "s/https:\/\/github.com\/minio\/minio-go\/releases\/tag\///") 19 | if [ -z "$MINIO_GO_VERSION" ]; then 20 | echo "unable to get minio-go version from github" 21 | exit 1 22 | fi 23 | 24 | test_run_dir="$MINT_RUN_CORE_DIR/minio-go" 25 | curl -sL -o "${test_run_dir}/main.go" "https://raw.githubusercontent.com/minio/minio-go/${MINIO_GO_VERSION}/functional_tests.go" 26 | (cd "$test_run_dir" && go mod tidy -compat=1.21 && CGO_ENABLED=0 go build --ldflags "-s -w" -o minio-go main.go) 27 | -------------------------------------------------------------------------------- /build/minio-java/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | SPOTBUGS_VERSION="4.2.2" ## needed since 8.0.2 release 19 | JUNIT_VERSION="4.12" ## JUNIT version 20 | MINIO_JAVA_VERSION=$(curl --retry 10 -s "https://repo1.maven.org/maven2/io/minio/minio/maven-metadata.xml" | sed -n "//{s/<.[^>]*>//g;p;q}" | sed "s/ *//g") 21 | if [ -z "$MINIO_JAVA_VERSION" ]; then 22 | echo "unable to get latest minio-java version from maven" 23 | exit 1 24 | fi 25 | 26 | test_run_dir="$MINT_RUN_CORE_DIR/minio-java" 27 | git clone --quiet https://github.com/minio/minio-java.git "$test_run_dir/minio-java.git" 28 | ( 29 | cd "$test_run_dir/minio-java.git" 30 | git checkout --quiet "tags/${MINIO_JAVA_VERSION}" 31 | ) 32 | $WGET --output-document="$test_run_dir/minio-${MINIO_JAVA_VERSION}-all.jar" "https://repo1.maven.org/maven2/io/minio/minio/${MINIO_JAVA_VERSION}/minio-${MINIO_JAVA_VERSION}-all.jar" 33 | $WGET --output-document="$test_run_dir/minio-admin-${MINIO_JAVA_VERSION}-all.jar" "https://repo1.maven.org/maven2/io/minio/minio-admin/${MINIO_JAVA_VERSION}/minio-admin-${MINIO_JAVA_VERSION}-all.jar" 34 | $WGET --output-document="$test_run_dir/spotbugs-annotations-${SPOTBUGS_VERSION}.jar" "https://repo1.maven.org/maven2/com/github/spotbugs/spotbugs-annotations/${SPOTBUGS_VERSION}/spotbugs-annotations-${SPOTBUGS_VERSION}.jar" 35 | $WGET --output-document="$test_run_dir/junit-${JUNIT_VERSION}.jar" "https://repo1.maven.org/maven2/junit/junit/${JUNIT_VERSION}/junit-${JUNIT_VERSION}.jar" 36 | javac -cp "$test_run_dir/minio-${MINIO_JAVA_VERSION}-all.jar:$test_run_dir/minio-admin-${MINIO_JAVA_VERSION}-all.jar:$test_run_dir/spotbugs-annotations-${SPOTBUGS_VERSION}.jar:$test_run_dir/junit-${JUNIT_VERSION}.jar" "${test_run_dir}/minio-java.git/functional"/*.java 37 | cp -a "${test_run_dir}/minio-java.git/functional"/*.class "$test_run_dir/" 38 | rm -fr "$test_run_dir/minio-java.git" 39 | -------------------------------------------------------------------------------- /build/minio-js/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # Checkout at /mint/test-run/minio-js/ 19 | # During run of the test copy it to the the /min/run/core/minio-js/minio-js 20 | 21 | install_path="./test-run/minio-js/" 22 | rm -rf $install_path 23 | 24 | git clone https://github.com/minio/minio-js.git $install_path 25 | 26 | cd $install_path || exit 0 27 | 28 | # Get new tags from remote 29 | git fetch --tags 30 | # Get latest tag name 31 | # shellcheck disable=SC2046 32 | LATEST=$(git describe --tags $(git rev-list --tags --max-count=1)) 33 | 34 | echo "Using minio-js RELEASE $LATEST" 35 | 36 | git checkout "${LATEST}" --force &>/dev/null 37 | 38 | npm install --quiet &>/dev/null 39 | -------------------------------------------------------------------------------- /build/minio-py/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Mint (C) 2017-2020 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | MINIO_PY_VERSION=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/minio/minio-py/releases/latest | sed "s/https:\/\/github.com\/minio\/minio-py\/releases\/tag\///") 19 | if [ -z "$MINIO_PY_VERSION" ]; then 20 | echo "unable to get minio-py version from github" 21 | exit 1 22 | fi 23 | 24 | test_run_dir="$MINT_RUN_CORE_DIR/minio-py" 25 | pip3 install --user faker 26 | pip3 install minio=="${MINIO_PY_VERSION}" 27 | $WGET --output-document="$test_run_dir/tests.py" "https://raw.githubusercontent.com/minio/minio-py/${MINIO_PY_VERSION}/tests/functional/tests.py" 28 | -------------------------------------------------------------------------------- /build/s3cmd/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # Always install the latest. 19 | python -m pip install s3cmd 20 | -------------------------------------------------------------------------------- /build/s3select/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Mint (C) 2020 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | python -m pip install minio 19 | -------------------------------------------------------------------------------- /build/versioning/bucket.go: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Mint, (C) 2021 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | package main 21 | 22 | import ( 23 | "errors" 24 | "math/rand" 25 | "strings" 26 | "time" 27 | 28 | "github.com/aws/aws-sdk-go/aws" 29 | "github.com/aws/aws-sdk-go/service/s3" 30 | ) 31 | 32 | // Tests bucket versioned bucket and get its versioning configuration to check 33 | func testMakeBucket() { 34 | s3Client.Config.Region = aws.String("us-east-1") 35 | 36 | // initialize logging params 37 | startTime := time.Now() 38 | function := "testCreateVersioningBucket" 39 | bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-") 40 | args := map[string]interface{}{ 41 | "bucketName": bucketName, 42 | } 43 | _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ 44 | Bucket: aws.String(bucketName), 45 | }) 46 | if err != nil { 47 | failureLog(function, args, startTime, "", "Versioning CreateBucket Failed", err).Fatal() 48 | return 49 | } 50 | defer cleanupBucket(bucketName, function, args, startTime) 51 | 52 | putVersioningInput := &s3.PutBucketVersioningInput{ 53 | Bucket: aws.String(bucketName), 54 | VersioningConfiguration: &s3.VersioningConfiguration{ 55 | MFADelete: aws.String("Disabled"), 56 | Status: aws.String("Enabled"), 57 | }, 58 | } 59 | 60 | _, err = s3Client.PutBucketVersioning(putVersioningInput) 61 | if err != nil { 62 | if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") { 63 | ignoreLog(function, args, startTime, "Versioning is not implemented").Info() 64 | return 65 | } 66 | failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal() 67 | return 68 | } 69 | 70 | getVersioningInput := &s3.GetBucketVersioningInput{ 71 | Bucket: aws.String(bucketName), 72 | } 73 | 74 | result, err := s3Client.GetBucketVersioning(getVersioningInput) 75 | if err != nil { 76 | failureLog(function, args, startTime, "", "Get Versioning failed", err).Fatal() 77 | return 78 | } 79 | 80 | if *result.Status != "Enabled" { 81 | failureLog(function, args, startTime, "", "Get Versioning status failed", errors.New("unexpected versioning status")).Fatal() 82 | } 83 | 84 | successLogger(function, args, startTime).Info() 85 | } 86 | -------------------------------------------------------------------------------- /build/versioning/delete.go: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Mint, (C) 2021 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | package main 21 | 22 | import ( 23 | "fmt" 24 | "io/ioutil" 25 | "math/rand" 26 | "strings" 27 | "time" 28 | 29 | "github.com/aws/aws-sdk-go/aws" 30 | "github.com/aws/aws-sdk-go/aws/awserr" 31 | "github.com/aws/aws-sdk-go/service/s3" 32 | ) 33 | 34 | func testDeleteObject() { 35 | startTime := time.Now() 36 | function := "testDeleteObject" 37 | bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-") 38 | object := "testObject" 39 | objectContent := "my object content" 40 | expiry := 1 * time.Minute 41 | args := map[string]interface{}{ 42 | "bucketName": bucket, 43 | "objectName": object, 44 | "expiry": expiry, 45 | } 46 | 47 | _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ 48 | Bucket: aws.String(bucket), 49 | }) 50 | if err != nil { 51 | failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() 52 | return 53 | } 54 | defer cleanupBucket(bucket, function, args, startTime) 55 | 56 | putVersioningInput := &s3.PutBucketVersioningInput{ 57 | Bucket: aws.String(bucket), 58 | VersioningConfiguration: &s3.VersioningConfiguration{ 59 | Status: aws.String("Enabled"), 60 | }, 61 | } 62 | 63 | _, err = s3Client.PutBucketVersioning(putVersioningInput) 64 | if err != nil { 65 | if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") { 66 | ignoreLog(function, args, startTime, "Versioning is not implemented").Info() 67 | return 68 | } 69 | failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal() 70 | return 71 | } 72 | 73 | putInput := &s3.PutObjectInput{ 74 | Body: aws.ReadSeekCloser(strings.NewReader(objectContent)), 75 | Bucket: aws.String(bucket), 76 | Key: aws.String(object), 77 | } 78 | 79 | putOutput, err := s3Client.PutObject(putInput) 80 | if err != nil { 81 | failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal() 82 | return 83 | } 84 | 85 | // First delete without version ID 86 | deleteInput := &s3.DeleteObjectInput{ 87 | Bucket: aws.String(bucket), 88 | Key: aws.String(object), 89 | } 90 | delOutput, err := s3Client.DeleteObject(deleteInput) 91 | if err != nil { 92 | failureLog(function, args, startTime, "", fmt.Sprintf("Delete expected to succeed but got %v", err), err).Fatal() 93 | return 94 | } 95 | 96 | // Get the delete marker version, should lead to an error 97 | getInput := &s3.GetObjectInput{ 98 | Bucket: aws.String(bucket), 99 | Key: aws.String(object), 100 | VersionId: aws.String(*delOutput.VersionId), 101 | } 102 | 103 | result, err := s3Client.GetObject(getInput) 104 | if err == nil { 105 | failureLog(function, args, startTime, "", "GetObject expected to fail but succeeded", nil).Fatal() 106 | return 107 | } 108 | if err != nil { 109 | aerr, ok := err.(awserr.Error) 110 | if !ok { 111 | failureLog(function, args, startTime, "", "GetObject unexpected error with delete marker", err).Fatal() 112 | return 113 | } 114 | if aerr.Code() != "MethodNotAllowed" { 115 | failureLog(function, args, startTime, "", "GetObject unexpected error with delete marker", err).Fatal() 116 | return 117 | } 118 | } 119 | 120 | // Get the older version, make sure it is preserved 121 | getInput = &s3.GetObjectInput{ 122 | Bucket: aws.String(bucket), 123 | Key: aws.String(object), 124 | VersionId: aws.String(*putOutput.VersionId), 125 | } 126 | 127 | result, err = s3Client.GetObject(getInput) 128 | if err != nil { 129 | failureLog(function, args, startTime, "", fmt.Sprintf("GetObject expected to succeed but failed with %v", err), err).Fatal() 130 | return 131 | } 132 | 133 | body, err := ioutil.ReadAll(result.Body) 134 | if err != nil { 135 | failureLog(function, args, startTime, "", fmt.Sprintf("GetObject expected to return data but failed with %v", err), err).Fatal() 136 | return 137 | } 138 | result.Body.Close() 139 | 140 | if string(body) != objectContent { 141 | failureLog(function, args, startTime, "", "GetObject unexpected body content", nil).Fatal() 142 | return 143 | } 144 | 145 | for i, versionID := range []string{*delOutput.VersionId, *putOutput.VersionId} { 146 | delInput := &s3.DeleteObjectInput{ 147 | Bucket: aws.String(bucket), 148 | Key: aws.String(object), 149 | VersionId: aws.String(versionID), 150 | } 151 | _, err := s3Client.DeleteObject(delInput) 152 | if err != nil { 153 | failureLog(function, args, startTime, "", fmt.Sprintf("DeleteObject (%d) expected to succeed but failed", i+1), err).Fatal() 154 | return 155 | } 156 | } 157 | 158 | listInput := &s3.ListObjectVersionsInput{ 159 | Bucket: aws.String(bucket), 160 | } 161 | 162 | listOutput, err := s3Client.ListObjectVersions(listInput) 163 | if err != nil { 164 | failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal() 165 | return 166 | } 167 | 168 | if len(listOutput.DeleteMarkers) != 0 || len(listOutput.CommonPrefixes) != 0 || len(listOutput.Versions) != 0 { 169 | failureLog(function, args, startTime, "", "ListObjectVersions returned some entries but expected to return nothing", nil).Fatal() 170 | return 171 | } 172 | 173 | successLogger(function, args, startTime).Info() 174 | } 175 | 176 | // Test deletion using multi delete API 177 | func testDeleteObjects() { 178 | startTime := time.Now() 179 | function := "testDeleteObjects" 180 | bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-") 181 | object := "testObject" 182 | objectContent := "my object content" 183 | expiry := 1 * time.Minute 184 | args := map[string]interface{}{ 185 | "bucketName": bucket, 186 | "objectName": object, 187 | "expiry": expiry, 188 | } 189 | 190 | _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ 191 | Bucket: aws.String(bucket), 192 | }) 193 | if err != nil { 194 | failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() 195 | return 196 | } 197 | defer cleanupBucket(bucket, function, args, startTime) 198 | 199 | putVersioningInput := &s3.PutBucketVersioningInput{ 200 | Bucket: aws.String(bucket), 201 | VersioningConfiguration: &s3.VersioningConfiguration{ 202 | Status: aws.String("Enabled"), 203 | }, 204 | } 205 | 206 | _, err = s3Client.PutBucketVersioning(putVersioningInput) 207 | if err != nil { 208 | if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") { 209 | ignoreLog(function, args, startTime, "Versioning is not implemented").Info() 210 | return 211 | } 212 | failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal() 213 | return 214 | } 215 | 216 | putInput := &s3.PutObjectInput{ 217 | Body: aws.ReadSeekCloser(strings.NewReader(objectContent)), 218 | Bucket: aws.String(bucket), 219 | Key: aws.String(object), 220 | } 221 | 222 | _, err = s3Client.PutObject(putInput) 223 | if err != nil { 224 | failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal() 225 | return 226 | } 227 | 228 | // First delete without version ID 229 | del := &s3.Delete{ 230 | Objects: []*s3.ObjectIdentifier{ 231 | { 232 | Key: aws.String(object), 233 | }, 234 | }, 235 | } 236 | deleteInput := &s3.DeleteObjectsInput{ 237 | Bucket: aws.String(bucket), 238 | Delete: del, 239 | } 240 | 241 | _, err = s3Client.DeleteObjects(deleteInput) 242 | if err != nil { 243 | failureLog(function, args, startTime, "", fmt.Sprintf("Delete expected to succeed but got %v", err), err).Fatal() 244 | return 245 | } 246 | 247 | // Check listing output: expected one version and one delete marker 248 | listInput := &s3.ListObjectVersionsInput{ 249 | Bucket: aws.String(bucket), 250 | } 251 | listOutput, err := s3Client.ListObjectVersions(listInput) 252 | if err != nil { 253 | failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal() 254 | return 255 | } 256 | if len(listOutput.DeleteMarkers) != 1 || len(listOutput.Versions) != 1 { 257 | failureLog(function, args, startTime, "", "ListObjectVersions returned unexpected result", nil).Fatal() 258 | return 259 | } 260 | 261 | // Delete all versions for the object 262 | del = &s3.Delete{ 263 | Objects: []*s3.ObjectIdentifier{ 264 | { 265 | Key: listOutput.DeleteMarkers[0].Key, 266 | VersionId: listOutput.DeleteMarkers[0].VersionId, 267 | }, 268 | { 269 | Key: listOutput.Versions[0].Key, 270 | VersionId: listOutput.Versions[0].VersionId, 271 | }, 272 | }, 273 | } 274 | deleteInput = &s3.DeleteObjectsInput{ 275 | Bucket: aws.String(bucket), 276 | Delete: del, 277 | } 278 | _, err = s3Client.DeleteObjects(deleteInput) 279 | if err != nil { 280 | failureLog(function, args, startTime, "", fmt.Sprintf("Delete expected to succeed but got %v", err), err).Fatal() 281 | return 282 | } 283 | 284 | // List and expect empty list 285 | listInput = &s3.ListObjectVersionsInput{ 286 | Bucket: aws.String(bucket), 287 | } 288 | listOutput, err = s3Client.ListObjectVersions(listInput) 289 | if err != nil { 290 | failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal() 291 | return 292 | } 293 | if len(listOutput.DeleteMarkers) != 0 || len(listOutput.Versions) != 0 { 294 | failureLog(function, args, startTime, "", "ListObjectVersions returned non empty result", nil).Fatal() 295 | return 296 | } 297 | 298 | successLogger(function, args, startTime).Info() 299 | } 300 | -------------------------------------------------------------------------------- /build/versioning/get.go: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Mint, (C) 2021 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | package main 21 | 22 | import ( 23 | "fmt" 24 | "io/ioutil" 25 | "math/rand" 26 | "strings" 27 | "time" 28 | 29 | "github.com/aws/aws-sdk-go/aws" 30 | "github.com/aws/aws-sdk-go/aws/awserr" 31 | "github.com/aws/aws-sdk-go/service/s3" 32 | ) 33 | 34 | // testGetObject tests all get object features - picking a particular 35 | // version id, check content and its metadata 36 | func testGetObject() { 37 | startTime := time.Now() 38 | function := "testGetObject" 39 | bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-") 40 | object := "testObject" 41 | expiry := 1 * time.Minute 42 | args := map[string]interface{}{ 43 | "bucketName": bucket, 44 | "objectName": object, 45 | "expiry": expiry, 46 | } 47 | 48 | _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ 49 | Bucket: aws.String(bucket), 50 | }) 51 | if err != nil { 52 | failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() 53 | return 54 | } 55 | defer cleanupBucket(bucket, function, args, startTime) 56 | 57 | putVersioningInput := &s3.PutBucketVersioningInput{ 58 | Bucket: aws.String(bucket), 59 | VersioningConfiguration: &s3.VersioningConfiguration{ 60 | Status: aws.String("Enabled"), 61 | }, 62 | } 63 | 64 | _, err = s3Client.PutBucketVersioning(putVersioningInput) 65 | if err != nil { 66 | if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") { 67 | ignoreLog(function, args, startTime, "Versioning is not implemented").Info() 68 | return 69 | } 70 | failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal() 71 | return 72 | } 73 | 74 | putInput1 := &s3.PutObjectInput{ 75 | Body: aws.ReadSeekCloser(strings.NewReader("my content 1")), 76 | Bucket: aws.String(bucket), 77 | Key: aws.String(object), 78 | } 79 | _, err = s3Client.PutObject(putInput1) 80 | if err != nil { 81 | failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal() 82 | return 83 | } 84 | putInput2 := &s3.PutObjectInput{ 85 | Body: aws.ReadSeekCloser(strings.NewReader("content file 2")), 86 | Bucket: aws.String(bucket), 87 | Key: aws.String(object), 88 | } 89 | _, err = s3Client.PutObject(putInput2) 90 | if err != nil { 91 | failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal() 92 | return 93 | } 94 | 95 | deleteInput := &s3.DeleteObjectInput{ 96 | Bucket: aws.String(bucket), 97 | Key: aws.String(object), 98 | } 99 | 100 | _, err = s3Client.DeleteObject(deleteInput) 101 | if err != nil { 102 | failureLog(function, args, startTime, "", fmt.Sprintf("Delete expected to succeed but got %v", err), err).Fatal() 103 | return 104 | } 105 | 106 | input := &s3.ListObjectVersionsInput{ 107 | Bucket: aws.String(bucket), 108 | } 109 | 110 | result, err := s3Client.ListObjectVersions(input) 111 | if err != nil { 112 | failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal() 113 | return 114 | } 115 | 116 | testCases := []struct { 117 | content string 118 | versionId string 119 | deleteMarker bool 120 | }{ 121 | {"", *(*result.DeleteMarkers[0]).VersionId, true}, 122 | {"content file 2", *(*result.Versions[0]).VersionId, false}, 123 | {"my content 1", *(*result.Versions[1]).VersionId, false}, 124 | } 125 | 126 | for i, testCase := range testCases { 127 | getInput := &s3.GetObjectInput{ 128 | Bucket: aws.String(bucket), 129 | Key: aws.String(object), 130 | VersionId: aws.String(testCase.versionId), 131 | } 132 | 133 | result, err := s3Client.GetObject(getInput) 134 | if testCase.deleteMarker && err == nil { 135 | failureLog(function, args, startTime, "", fmt.Sprintf("GetObject(%d) expected to fail but succeeded", i+1), nil).Fatal() 136 | return 137 | } 138 | 139 | if !testCase.deleteMarker && err != nil { 140 | failureLog(function, args, startTime, "", fmt.Sprintf("GetObject(%d) expected to succeed but failed", i+1), err).Fatal() 141 | return 142 | } 143 | 144 | if testCase.deleteMarker { 145 | aerr, ok := err.(awserr.Error) 146 | if !ok { 147 | failureLog(function, args, startTime, "", fmt.Sprintf("GetObject(%d) unexpected error with delete marker", i+1), err).Fatal() 148 | return 149 | } 150 | if aerr.Code() != "MethodNotAllowed" { 151 | failureLog(function, args, startTime, "", fmt.Sprintf("GetObject(%d) unexpected error with delete marker", i+1), err).Fatal() 152 | return 153 | } 154 | continue 155 | } 156 | 157 | body, err := ioutil.ReadAll(result.Body) 158 | if err != nil { 159 | failureLog(function, args, startTime, "", fmt.Sprintf("GetObject(%d) expected to return data but failed", i+1), err).Fatal() 160 | return 161 | } 162 | result.Body.Close() 163 | 164 | if string(body) != testCase.content { 165 | failureLog(function, args, startTime, "", fmt.Sprintf("GetObject(%d) unexpected body content", i+1), err).Fatal() 166 | return 167 | } 168 | } 169 | 170 | successLogger(function, args, startTime).Info() 171 | } 172 | -------------------------------------------------------------------------------- /build/versioning/go.mod: -------------------------------------------------------------------------------- 1 | module mint.minio.io/versioning/tests 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/aws/aws-sdk-go v1.44.257 7 | github.com/sirupsen/logrus v1.9.0 8 | ) 9 | 10 | require ( 11 | github.com/jmespath/go-jmespath v0.4.0 // indirect 12 | golang.org/x/sys v0.5.0 // indirect 13 | ) 14 | -------------------------------------------------------------------------------- /build/versioning/go.sum: -------------------------------------------------------------------------------- 1 | github.com/aws/aws-sdk-go v1.44.257 h1:HwelXYZZ8c34uFFhgVw3ybu2gB5fkk8KLj2idTvzZb8= 2 | github.com/aws/aws-sdk-go v1.44.257/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= 3 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 5 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= 7 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 8 | github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= 9 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 10 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 11 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 12 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 13 | github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= 14 | github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 15 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 16 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 17 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 18 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 19 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 20 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 21 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 22 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 23 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 24 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 25 | golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= 26 | golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= 27 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 28 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 29 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 30 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 31 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 32 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 33 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 34 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 35 | golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 36 | golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= 37 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 38 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 39 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 40 | golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 41 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 42 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 43 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 44 | golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= 45 | golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 46 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 47 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 48 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 49 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 50 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 51 | gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= 52 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 53 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 54 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 55 | -------------------------------------------------------------------------------- /build/versioning/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Mint (C) 2017-2021 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | test_run_dir="$MINT_RUN_CORE_DIR/versioning" 19 | test_build_dir="$MINT_RUN_BUILD_DIR/versioning" 20 | 21 | (cd "$test_build_dir" && CGO_ENABLED=0 go build --ldflags "-s -w" -o "$test_run_dir/tests") 22 | -------------------------------------------------------------------------------- /build/versioning/main.go: -------------------------------------------------------------------------------- 1 | // Mint, (C) 2021 Minio, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package main 17 | 18 | import ( 19 | "os" 20 | "time" 21 | 22 | "github.com/aws/aws-sdk-go/aws" 23 | "github.com/aws/aws-sdk-go/aws/credentials" 24 | "github.com/aws/aws-sdk-go/aws/session" 25 | "github.com/aws/aws-sdk-go/service/s3" 26 | log "github.com/sirupsen/logrus" 27 | ) 28 | 29 | // S3 client for testing 30 | var s3Client *s3.S3 31 | 32 | func cleanupBucket(bucket string, function string, args map[string]interface{}, startTime time.Time) { 33 | start := time.Now() 34 | 35 | input := &s3.ListObjectVersionsInput{ 36 | Bucket: aws.String(bucket), 37 | } 38 | 39 | for time.Since(start) < 30*time.Minute { 40 | err := s3Client.ListObjectVersionsPages(input, 41 | func(page *s3.ListObjectVersionsOutput, lastPage bool) bool { 42 | for _, v := range page.Versions { 43 | input := &s3.DeleteObjectInput{ 44 | Bucket: &bucket, 45 | Key: v.Key, 46 | VersionId: v.VersionId, 47 | BypassGovernanceRetention: aws.Bool(true), 48 | } 49 | _, err := s3Client.DeleteObject(input) 50 | if err != nil { 51 | return true 52 | } 53 | } 54 | for _, v := range page.DeleteMarkers { 55 | input := &s3.DeleteObjectInput{ 56 | Bucket: &bucket, 57 | Key: v.Key, 58 | VersionId: v.VersionId, 59 | BypassGovernanceRetention: aws.Bool(true), 60 | } 61 | _, err := s3Client.DeleteObject(input) 62 | if err != nil { 63 | return true 64 | } 65 | } 66 | return true 67 | }) 68 | 69 | _, err = s3Client.DeleteBucket(&s3.DeleteBucketInput{ 70 | Bucket: aws.String(bucket), 71 | }) 72 | if err != nil { 73 | time.Sleep(30 * time.Second) 74 | continue 75 | } 76 | return 77 | } 78 | 79 | failureLog(function, args, startTime, "", "Unable to cleanup bucket after compliance tests", nil).Fatal() 80 | return 81 | } 82 | 83 | func main() { 84 | endpoint := os.Getenv("SERVER_ENDPOINT") 85 | accessKey := os.Getenv("ACCESS_KEY") 86 | secretKey := os.Getenv("SECRET_KEY") 87 | secure := os.Getenv("ENABLE_HTTPS") 88 | sdkEndpoint := "http://" + endpoint 89 | if secure == "1" { 90 | sdkEndpoint = "https://" + endpoint 91 | } 92 | 93 | creds := credentials.NewStaticCredentials(accessKey, secretKey, "") 94 | newSession := session.New() 95 | s3Config := &aws.Config{ 96 | Credentials: creds, 97 | Endpoint: aws.String(sdkEndpoint), 98 | Region: aws.String("us-east-1"), 99 | S3ForcePathStyle: aws.Bool(true), 100 | } 101 | 102 | // Create an S3 service object in the default region. 103 | s3Client = s3.New(newSession, s3Config) 104 | 105 | // Output to stdout instead of the default stderr 106 | log.SetOutput(os.Stdout) 107 | // create custom formatter 108 | mintFormatter := mintJSONFormatter{} 109 | // set custom formatter 110 | log.SetFormatter(&mintFormatter) 111 | // log Info or above -- success cases are Info level, failures are Fatal level 112 | log.SetLevel(log.InfoLevel) 113 | 114 | testMakeBucket() 115 | testPutObject() 116 | testPutObjectWithTaggingAndMetadata() 117 | testGetObject() 118 | testStatObject() 119 | testDeleteObject() 120 | testDeleteObjects() 121 | testListObjectVersionsSimple() 122 | testListObjectVersionsWithPrefixAndDelimiter() 123 | testListObjectVersionsKeysContinuation() 124 | testListObjectVersionsVersionIDContinuation() 125 | testListObjectsVersionsWithEmptyDirObject() 126 | testTagging() 127 | testLockingLegalhold() 128 | testPutGetRetentionCompliance() 129 | testPutGetDeleteRetentionGovernance() 130 | testLockingRetentionGovernance() 131 | testLockingRetentionCompliance() 132 | } 133 | -------------------------------------------------------------------------------- /build/versioning/stat.go: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Mint, (C) 2021 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | package main 21 | 22 | import ( 23 | "fmt" 24 | "math/rand" 25 | "strings" 26 | "time" 27 | 28 | "github.com/aws/aws-sdk-go/aws" 29 | "github.com/aws/aws-sdk-go/aws/awserr" 30 | "github.com/aws/aws-sdk-go/service/s3" 31 | ) 32 | 33 | func testStatObject() { 34 | startTime := time.Now() 35 | function := "testStatObject" 36 | bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-") 37 | object := "testObject" 38 | expiry := 1 * time.Minute 39 | args := map[string]interface{}{ 40 | "bucketName": bucket, 41 | "objectName": object, 42 | "expiry": expiry, 43 | } 44 | 45 | _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ 46 | Bucket: aws.String(bucket), 47 | }) 48 | if err != nil { 49 | failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() 50 | return 51 | } 52 | defer cleanupBucket(bucket, function, args, startTime) 53 | 54 | putVersioningInput := &s3.PutBucketVersioningInput{ 55 | Bucket: aws.String(bucket), 56 | VersioningConfiguration: &s3.VersioningConfiguration{ 57 | Status: aws.String("Enabled"), 58 | }, 59 | } 60 | 61 | _, err = s3Client.PutBucketVersioning(putVersioningInput) 62 | if err != nil { 63 | if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") { 64 | ignoreLog(function, args, startTime, "Versioning is not implemented").Info() 65 | return 66 | } 67 | failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal() 68 | return 69 | } 70 | 71 | putInput1 := &s3.PutObjectInput{ 72 | Body: aws.ReadSeekCloser(strings.NewReader("my content 1")), 73 | Bucket: aws.String(bucket), 74 | Key: aws.String(object), 75 | } 76 | _, err = s3Client.PutObject(putInput1) 77 | if err != nil { 78 | failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal() 79 | return 80 | } 81 | putInput2 := &s3.PutObjectInput{ 82 | Body: aws.ReadSeekCloser(strings.NewReader("content file 2")), 83 | Bucket: aws.String(bucket), 84 | Key: aws.String(object), 85 | } 86 | _, err = s3Client.PutObject(putInput2) 87 | if err != nil { 88 | failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal() 89 | return 90 | } 91 | 92 | deleteInput := &s3.DeleteObjectInput{ 93 | Bucket: aws.String(bucket), 94 | Key: aws.String(object), 95 | } 96 | 97 | _, err = s3Client.DeleteObject(deleteInput) 98 | if err != nil { 99 | failureLog(function, args, startTime, "", fmt.Sprintf("Delete expected to succeed but got %v", err), err).Fatal() 100 | return 101 | } 102 | 103 | input := &s3.ListObjectVersionsInput{ 104 | Bucket: aws.String(bucket), 105 | } 106 | 107 | result, err := s3Client.ListObjectVersions(input) 108 | if err != nil { 109 | failureLog(function, args, startTime, "", fmt.Sprintf("ListObjectVersions expected to succeed but got %v", err), err).Fatal() 110 | return 111 | } 112 | 113 | testCases := []struct { 114 | size int64 115 | versionId string 116 | contentType string 117 | deleteMarker bool 118 | }{ 119 | {0, *(*result.DeleteMarkers[0]).VersionId, "", true}, 120 | {14, *(*result.Versions[0]).VersionId, "binary/octet-stream", false}, 121 | {12, *(*result.Versions[1]).VersionId, "binary/octet-stream", false}, 122 | } 123 | 124 | for i, testCase := range testCases { 125 | headInput := &s3.HeadObjectInput{ 126 | Bucket: aws.String(bucket), 127 | Key: aws.String(object), 128 | VersionId: aws.String(testCase.versionId), 129 | } 130 | 131 | result, err := s3Client.HeadObject(headInput) 132 | if testCase.deleteMarker && err == nil { 133 | failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) expected to fail but succeeded", i+1), nil).Fatal() 134 | return 135 | } 136 | 137 | if !testCase.deleteMarker && err != nil { 138 | failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) expected to succeed but failed", i+1), err).Fatal() 139 | return 140 | } 141 | 142 | if testCase.deleteMarker { 143 | aerr, ok := err.(awserr.Error) 144 | if !ok { 145 | failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) unexpected error with delete marker", i+1), err).Fatal() 146 | return 147 | } 148 | if aerr.Code() != "MethodNotAllowed" { 149 | failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) unexpected error code with delete marker", i+1), err).Fatal() 150 | return 151 | } 152 | continue 153 | } 154 | 155 | if *result.ContentLength != testCase.size { 156 | failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) unexpected Content-Length", i+1), err).Fatal() 157 | return 158 | } 159 | 160 | if !etagRegex.MatchString(*result.ETag) { 161 | failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) unexpected ETag", i+1), err).Fatal() 162 | return 163 | } 164 | 165 | if *result.ContentType != testCase.contentType { 166 | failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) unexpected Content-Type", i+1), err).Fatal() 167 | return 168 | } 169 | 170 | if result.DeleteMarker != nil && *result.DeleteMarker { 171 | failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) unexpected DeleteMarker", i+1), err).Fatal() 172 | return 173 | } 174 | 175 | if time.Since(*result.LastModified) > time.Hour { 176 | failureLog(function, args, startTime, "", fmt.Sprintf("StatObject (%d) unexpected LastModified", i+1), err).Fatal() 177 | return 178 | } 179 | } 180 | 181 | successLogger(function, args, startTime).Info() 182 | } 183 | -------------------------------------------------------------------------------- /build/versioning/tagging.go: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Mint, (C) 2021 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | package main 21 | 22 | import ( 23 | "fmt" 24 | "math/rand" 25 | "reflect" 26 | "strings" 27 | "time" 28 | 29 | "github.com/aws/aws-sdk-go/aws" 30 | "github.com/aws/aws-sdk-go/service/s3" 31 | ) 32 | 33 | // Test PUT/GET/DELETE tagging for separate versions 34 | func testTagging() { 35 | startTime := time.Now() 36 | function := "testTagging" 37 | bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "versioning-test-") 38 | object := "testObject" 39 | expiry := 1 * time.Minute 40 | args := map[string]interface{}{ 41 | "bucketName": bucket, 42 | "objectName": object, 43 | "expiry": expiry, 44 | } 45 | 46 | _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ 47 | Bucket: aws.String(bucket), 48 | }) 49 | if err != nil { 50 | failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() 51 | return 52 | } 53 | defer cleanupBucket(bucket, function, args, startTime) 54 | 55 | putVersioningInput := &s3.PutBucketVersioningInput{ 56 | Bucket: aws.String(bucket), 57 | VersioningConfiguration: &s3.VersioningConfiguration{ 58 | Status: aws.String("Enabled"), 59 | }, 60 | } 61 | 62 | _, err = s3Client.PutBucketVersioning(putVersioningInput) 63 | if err != nil { 64 | if strings.Contains(err.Error(), "NotImplemented: A header you provided implies functionality that is not implemented") { 65 | ignoreLog(function, args, startTime, "Versioning is not implemented").Info() 66 | return 67 | } 68 | failureLog(function, args, startTime, "", "Put versioning failed", err).Fatal() 69 | return 70 | } 71 | 72 | type uploadedObject struct { 73 | content string 74 | tagging []*s3.Tag 75 | versionId string 76 | deleteMarker bool 77 | } 78 | 79 | uploads := []uploadedObject{ 80 | {content: "my content 1", tagging: []*s3.Tag{{Key: aws.String("type"), Value: aws.String("text")}}}, 81 | {content: "content file 2"}, 82 | {content: "\"%32&é", tagging: []*s3.Tag{{Key: aws.String("type"), Value: aws.String("garbage")}}}, 83 | {deleteMarker: true}, 84 | } 85 | 86 | // Upload versions and save their version IDs 87 | for i := range uploads { 88 | if uploads[i].deleteMarker { 89 | // Delete the current object to create a delete marker) 90 | deleteInput := &s3.DeleteObjectInput{ 91 | Bucket: aws.String(bucket), 92 | Key: aws.String(object), 93 | } 94 | deleteOutput, err := s3Client.DeleteObject(deleteInput) 95 | if err != nil { 96 | failureLog(function, args, startTime, "", fmt.Sprintf("DELETE object expected to succeed but got %v", err), err).Fatal() 97 | return 98 | } 99 | uploads[i].versionId = *deleteOutput.VersionId 100 | continue 101 | } 102 | 103 | putInput := &s3.PutObjectInput{ 104 | Body: aws.ReadSeekCloser(strings.NewReader(uploads[i].content)), 105 | Bucket: aws.String(bucket), 106 | Key: aws.String(object), 107 | } 108 | output, err := s3Client.PutObject(putInput) 109 | if err != nil { 110 | failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal() 111 | return 112 | } 113 | uploads[i].versionId = *output.VersionId 114 | } 115 | 116 | // Put tagging on each version 117 | for i := range uploads { 118 | if uploads[i].tagging == nil { 119 | continue 120 | } 121 | putTaggingInput := &s3.PutObjectTaggingInput{ 122 | Bucket: aws.String(bucket), 123 | Key: aws.String(object), 124 | Tagging: &s3.Tagging{TagSet: uploads[i].tagging}, 125 | VersionId: aws.String(uploads[i].versionId), 126 | } 127 | _, err = s3Client.PutObjectTagging(putTaggingInput) 128 | if err != nil { 129 | failureLog(function, args, startTime, "", fmt.Sprintf("PUT Object tagging expected to succeed but got %v", err), err).Fatal() 130 | return 131 | } 132 | } 133 | 134 | // Check versions tagging 135 | for i := range uploads { 136 | input := &s3.GetObjectTaggingInput{ 137 | Bucket: aws.String(bucket), 138 | Key: aws.String(object), 139 | VersionId: aws.String(uploads[i].versionId), 140 | } 141 | result, err := s3Client.GetObjectTagging(input) 142 | if err == nil && uploads[i].deleteMarker { 143 | failureLog(function, args, startTime, "", "GET Object tagging expected to fail with delete marker but succeded", err).Fatal() 144 | return 145 | } 146 | if err != nil && !uploads[i].deleteMarker { 147 | failureLog(function, args, startTime, "", fmt.Sprintf("GET Object tagging expected to succeed but got %v", err), err).Fatal() 148 | return 149 | } 150 | 151 | if uploads[i].deleteMarker { 152 | continue 153 | } 154 | 155 | if !reflect.DeepEqual(result.TagSet, uploads[i].tagging) { 156 | failureLog(function, args, startTime, "", "GET Object tagging returned unexpected result", nil).Fatal() 157 | return 158 | } 159 | } 160 | 161 | // Remove all tagging for all objects 162 | for i := range uploads { 163 | input := &s3.DeleteObjectTaggingInput{ 164 | Bucket: aws.String(bucket), 165 | Key: aws.String(object), 166 | VersionId: aws.String(uploads[i].versionId), 167 | } 168 | _, err := s3Client.DeleteObjectTagging(input) 169 | if err == nil && uploads[i].deleteMarker { 170 | failureLog(function, args, startTime, "", "DELETE Object tagging expected to fail with delete marker but succeded", err).Fatal() 171 | return 172 | } 173 | if err != nil && !uploads[i].deleteMarker { 174 | failureLog(function, args, startTime, "", fmt.Sprintf("GET Object tagging expected to succeed but got %v", err), err).Fatal() 175 | return 176 | } 177 | } 178 | 179 | // Check for tagging after removal 180 | for i := range uploads { 181 | if uploads[i].deleteMarker { 182 | // Avoid testing this use case since already tested earlier 183 | continue 184 | } 185 | input := &s3.GetObjectTaggingInput{ 186 | Bucket: aws.String(bucket), 187 | Key: aws.String(object), 188 | VersionId: aws.String(uploads[i].versionId), 189 | } 190 | result, err := s3Client.GetObjectTagging(input) 191 | if err != nil { 192 | failureLog(function, args, startTime, "", fmt.Sprintf("GET Object tagging expected to succeed but got %v", err), err).Fatal() 193 | return 194 | } 195 | var nilTagSet []*s3.Tag 196 | if !reflect.DeepEqual(result.TagSet, nilTagSet) { 197 | failureLog(function, args, startTime, "", "GET Object tagging after DELETE returned unexpected result", nil).Fatal() 198 | return 199 | } 200 | } 201 | 202 | successLogger(function, args, startTime).Info() 203 | } 204 | -------------------------------------------------------------------------------- /build/versioning/utils.go: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Mint, (C) 2021 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | package main 21 | 22 | import ( 23 | "encoding/json" 24 | "encoding/xml" 25 | "fmt" 26 | "math/rand" 27 | "net/http" 28 | "runtime" 29 | "strings" 30 | "time" 31 | 32 | log "github.com/sirupsen/logrus" 33 | ) 34 | 35 | const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" 36 | const ( 37 | letterIdxBits = 6 // 6 bits to represent a letter index 38 | letterIdxMask = 1<= 0; { 135 | if remain == 0 { 136 | cache, remain = src.Int63(), letterIdxMax 137 | } 138 | if idx := int(cache & letterIdxMask); idx < len(letterBytes) { 139 | b[i] = letterBytes[idx] 140 | i-- 141 | } 142 | cache >>= letterIdxBits 143 | remain-- 144 | } 145 | return prefix + string(b[0:30-len(prefix)]) 146 | } 147 | -------------------------------------------------------------------------------- /create-data-files.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | MINT_DATA_DIR="$MINT_ROOT_DIR/data" 19 | 20 | declare -A data_file_map 21 | data_file_map["datafile-0-b"]="0" 22 | data_file_map["datafile-1-b"]="1" 23 | data_file_map["datafile-1-kB"]="1K" 24 | data_file_map["datafile-10-kB"]="10K" 25 | data_file_map["datafile-33-kB"]="33K" 26 | data_file_map["datafile-100-kB"]="100K" 27 | data_file_map["datafile-1.03-MB"]="1056K" 28 | data_file_map["datafile-1-MB"]="1M" 29 | data_file_map["datafile-5-MB"]="5M" 30 | data_file_map["datafile-5243880-b"]="5243880" 31 | data_file_map["datafile-6-MB"]="6M" 32 | data_file_map["datafile-10-MB"]="10M" 33 | data_file_map["datafile-11-MB"]="11M" 34 | data_file_map["datafile-65-MB"]="65M" 35 | data_file_map["datafile-129-MB"]="129M" 36 | 37 | mkdir -p "$MINT_DATA_DIR" 38 | for filename in "${!data_file_map[@]}"; do 39 | echo "creating $MINT_DATA_DIR/$filename" 40 | if ! shred -n 1 -s "${data_file_map[$filename]}" - 1>"$MINT_DATA_DIR/$filename" 2>/dev/null; then 41 | echo "unable to create data file $MINT_DATA_DIR/$filename" 42 | exit 1 43 | fi 44 | done 45 | -------------------------------------------------------------------------------- /entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | ./mint.sh "$@" & 19 | 20 | # Get the pid to be used for kill command if required 21 | main_pid="$!" 22 | trap 'echo -e "\nAborting Mint..."; kill $main_pid' SIGINT SIGTERM 23 | # use -n here to catch mint.sh exit code, notify to ci 24 | wait -n 25 | -------------------------------------------------------------------------------- /install-packages.list: -------------------------------------------------------------------------------- 1 | git 2 | python3-pip 3 | nodejs 4 | openjdk-8-jdk 5 | openjdk-8-jdk-headless 6 | dirmngr 7 | apt-transport-https 8 | dotnet-sdk-6.0 9 | ca-certificates-mono 10 | libunwind8 11 | ruby 12 | ruby-dev 13 | ruby-bundler 14 | php 15 | php-curl 16 | php-xml 17 | ant 18 | -------------------------------------------------------------------------------- /mint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2017-2022 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | CONTAINER_ID=$(grep -o -e '[0-f]\{12,\}' /proc/1/cpuset | awk '{print substr($1, 1, 12)}') 19 | MINT_DATA_DIR=${MINT_DATA_DIR:-/mint/data} 20 | MINT_MODE=${MINT_MODE:-core} 21 | SERVER_REGION=${SERVER_REGION:-us-east-1} 22 | ENABLE_HTTPS=${ENABLE_HTTPS:-0} 23 | ENABLE_VIRTUAL_STYLE=${ENABLE_VIRTUAL_STYLE:-0} 24 | RUN_ON_FAIL=${RUN_ON_FAIL:-0} 25 | 26 | if [ -z "$SERVER_ENDPOINT" ]; then 27 | SERVER_ENDPOINT="play.minio.io:9000" 28 | ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" 29 | SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" 30 | ENABLE_HTTPS=1 31 | fi 32 | 33 | if [ "$ENABLE_VIRTUAL_STYLE" -eq 1 ]; then 34 | SERVER_IP="${SERVER_ENDPOINT%%:*}" 35 | SERVER_PORT="${SERVER_ENDPOINT/*:/}" 36 | # Check if SERVER_IP is actually IPv4 address 37 | IFS=. read -ra octets <<<"$SERVER_IP" 38 | if [ "${#octets[@]}" -ne 4 ]; then 39 | echo "$SERVER_IP must be an IP address" 40 | exit 1 41 | fi 42 | for octet in "${octets[@]}"; do 43 | if [ "$octet" -lt 0 ] 2>/dev/null || [ "$octet" -gt 255 ] 2>/dev/null; then 44 | echo "$SERVER_IP must be an IP address" 45 | exit 1 46 | fi 47 | done 48 | fi 49 | 50 | ROOT_DIR="$PWD" 51 | TESTS_DIR="$ROOT_DIR/run/core" 52 | 53 | BASE_LOG_DIR="$ROOT_DIR/log" 54 | LOG_FILE="log.json" 55 | ERROR_FILE="error.log" 56 | mkdir -p "$BASE_LOG_DIR" 57 | 58 | function humanize_time() { 59 | time="$1" 60 | days=$((time / 60 / 60 / 24)) 61 | hours=$((time / 60 / 60 % 24)) 62 | minutes=$((time / 60 % 60)) 63 | seconds=$((time % 60)) 64 | 65 | ((days > 0)) && echo -n "$days days " 66 | ((hours > 0)) && echo -n "$hours hours " 67 | ((minutes > 0)) && echo -n "$minutes minutes " 68 | ((days > 0 || hours > 0 || minutes > 0)) && echo -n "and " 69 | echo "$seconds seconds" 70 | } 71 | 72 | function run_test() { 73 | if [ ! -d "$1" ]; then 74 | return 1 75 | fi 76 | 77 | start=$(date +%s) 78 | 79 | mkdir -p "$BASE_LOG_DIR/$sdk_name" 80 | 81 | (cd "$sdk_dir" && ./run.sh "$BASE_LOG_DIR/$LOG_FILE" "$BASE_LOG_DIR/$sdk_name/$ERROR_FILE") 82 | rv=$? 83 | end=$(date +%s) 84 | duration=$(humanize_time $((end - start))) 85 | 86 | if [ "$rv" -eq 0 ]; then 87 | echo "done in $duration" 88 | else 89 | echo "FAILED in $duration" 90 | entry=$(tail -n 1 "$BASE_LOG_DIR/$LOG_FILE") 91 | status=$(jq -e -r .status <<<"$entry") 92 | jq_rv=$? 93 | if [ "$jq_rv" -ne 0 ]; then 94 | echo "$entry" 95 | fi 96 | ## Show error.log when status is empty or not "FAIL". 97 | ## This may happen when test run failed without providing logs. 98 | if [ "$jq_rv" -ne 0 ] || [ -z "$status" ] || { [ "$status" != "FAIL" ] && [ "$status" != "fail" ]; }; then 99 | cat "$BASE_LOG_DIR/$sdk_name/$ERROR_FILE" 100 | else 101 | jq . <<<"$entry" 102 | fi 103 | fi 104 | return $rv 105 | } 106 | 107 | function trust_s3_endpoint_tls_cert() { 108 | # Download the public certificate from the server 109 | openssl s_client -showcerts -verify 5 -connect "$SERVER_ENDPOINT" out}' 111 | for cert in *.pem; do 112 | cat "${cert}" >>/etc/ssl/certs/ca-certificates.crt 113 | done 114 | 115 | # Ask different SDKs/tools to load system certificates 116 | export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt 117 | export NODE_EXTRA_CA_CERTS=/etc/ssl/certs/ca-certificates.crt 118 | export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt 119 | export AWS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt 120 | } 121 | 122 | function main() { 123 | export MINT_DATA_DIR 124 | export MINT_MODE 125 | export SERVER_ENDPOINT 126 | export SERVER_IP 127 | export SERVER_PORT 128 | 129 | export ACCESS_KEY 130 | export SECRET_KEY 131 | export ENABLE_HTTPS 132 | export SERVER_REGION 133 | export ENABLE_VIRTUAL_STYLE 134 | export RUN_ON_FAIL 135 | 136 | echo "Running with" 137 | echo "SERVER_ENDPOINT: $SERVER_ENDPOINT" 138 | echo "ACCESS_KEY: $ACCESS_KEY" 139 | echo "SECRET_KEY: ***REDACTED***" 140 | echo "ENABLE_HTTPS: $ENABLE_HTTPS" 141 | echo "SERVER_REGION: $SERVER_REGION" 142 | echo "MINT_DATA_DIR: $MINT_DATA_DIR" 143 | echo "MINT_MODE: $MINT_MODE" 144 | echo "ENABLE_VIRTUAL_STYLE: $ENABLE_VIRTUAL_STYLE" 145 | echo "RUN_ON_FAIL: $RUN_ON_FAIL" 146 | echo 147 | echo "To get logs, run 'docker cp ${CONTAINER_ID}:/mint/log /tmp/mint-logs'" 148 | echo 149 | 150 | [ "$ENABLE_HTTPS" == "1" ] && trust_s3_endpoint_tls_cert 151 | 152 | declare -a run_list 153 | sdks=("$@") 154 | 155 | if [ "$#" -eq 0 ]; then 156 | cd "$TESTS_DIR" || exit 157 | sdks=(*) 158 | cd .. || exit 159 | fi 160 | 161 | for sdk in "${sdks[@]}"; do 162 | sdk=$(basename "$sdk") 163 | run_list=("${run_list[@]}" "$TESTS_DIR/$sdk") 164 | done 165 | 166 | count="${#run_list[@]}" 167 | i=0 168 | j=0 169 | for sdk_dir in "${run_list[@]}"; do 170 | sdk_name=$(basename "$sdk_dir") 171 | ((i++)) 172 | ((j++)) 173 | if [ ! -d "$sdk_dir" ]; then 174 | echo "Test $sdk_name not found. Exiting Mint." 175 | exit 1 176 | fi 177 | echo -n "($j/$count) Running $sdk_name tests ... " 178 | if ! run_test "$sdk_dir"; then 179 | ((i--)) 180 | fi 181 | done 182 | 183 | ## Report when all tests in run_list are run 184 | if [ "$i" -eq "$count" ]; then 185 | echo -e "\nAll tests ran successfully" 186 | else 187 | echo -e "\nExecuted $i out of $count tests successfully." 188 | exit 1 189 | fi 190 | } 191 | main "$@" 192 | -------------------------------------------------------------------------------- /postinstall.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | export APT="apt --quiet --yes" 19 | 20 | # remove all packages listed in remove-packages.list 21 | xargs --arg-file="${MINT_ROOT_DIR}/remove-packages.list" apt --quiet --yes purge 22 | ${APT} autoremove 23 | 24 | # flush to disk 25 | sync 26 | -------------------------------------------------------------------------------- /preinstall.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Mint (C) 2017-2022 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | source "${MINT_ROOT_DIR}"/source.sh 19 | 20 | # install nodejs source list 21 | if ! $WGET --output-document=- https://deb.nodesource.com/setup_20.x | bash -; then 22 | echo "unable to set nodejs repository" 23 | exit 1 24 | fi 25 | 26 | $APT install apt-transport-https 27 | 28 | if ! $WGET --output-document=packages-microsoft-prod.deb https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb | bash -; then 29 | echo "unable to download dotnet packages" 30 | exit 1 31 | fi 32 | 33 | dpkg -i packages-microsoft-prod.deb 34 | rm -f packages-microsoft-prod.deb 35 | 36 | $APT update 37 | $APT install gnupg ca-certificates unzip busybox 38 | 39 | # download and install golang 40 | download_url="https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz" 41 | if ! $WGET --output-document=- "$download_url" | tar -C "${GO_INSTALL_PATH}" -zxf -; then 42 | echo "unable to install go$GO_VERSION" 43 | exit 1 44 | fi 45 | 46 | xargs --arg-file="${MINT_ROOT_DIR}/install-packages.list" apt --quiet --yes install 47 | 48 | # set python 3.10 as default 49 | update-alternatives --install /usr/bin/python python /usr/bin/python3.10 1 50 | 51 | mkdir -p ${GRADLE_INSTALL_PATH} 52 | gradle_url="https://services.gradle.org/distributions/gradle-${GRADLE_VERSION}-bin.zip" 53 | if ! $WGET --output-document=- "$gradle_url" | busybox unzip -qq -d ${GRADLE_INSTALL_PATH} -; then 54 | echo "unable to install gradle-${GRADLE_VERSION}" 55 | exit 1 56 | fi 57 | 58 | chmod +x -v ${GRADLE_INSTALL_PATH}/gradle-${GRADLE_VERSION}/bin/gradle 59 | 60 | sync 61 | -------------------------------------------------------------------------------- /release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Minio Cloud Storage, (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | export MINT_ROOT_DIR=${MINT_ROOT_DIR:-/mint} 19 | source "${MINT_ROOT_DIR}"/source.sh 20 | 21 | # install mint app packages 22 | for pkg in "$MINT_ROOT_DIR/build"/*/install.sh; do 23 | echo "Running $pkg" 24 | $pkg 25 | done 26 | 27 | "${MINT_ROOT_DIR}"/postinstall.sh 28 | -------------------------------------------------------------------------------- /remove-packages.list: -------------------------------------------------------------------------------- 1 | wget 2 | git 3 | python3-pip 4 | ruby-dev 5 | ruby-bundler 6 | openjdk-8-jdk 7 | ant 8 | dotnet 9 | -------------------------------------------------------------------------------- /run/core/.minio-dotnet/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | /mint/run/core/minio-dotnet/out/Minio.Functional.Tests 1>>"$output_log_file" 2>"$error_log_file" 27 | -------------------------------------------------------------------------------- /run/core/aws-sdk-go/go.mod: -------------------------------------------------------------------------------- 1 | module mint.minio.io/aws-sdk-go 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/aws/aws-sdk-go v1.44.257 7 | github.com/sirupsen/logrus v1.9.0 8 | ) 9 | 10 | require ( 11 | github.com/jmespath/go-jmespath v0.4.0 // indirect 12 | golang.org/x/sys v0.5.0 // indirect 13 | ) 14 | -------------------------------------------------------------------------------- /run/core/aws-sdk-go/go.sum: -------------------------------------------------------------------------------- 1 | github.com/aws/aws-sdk-go v1.44.257 h1:HwelXYZZ8c34uFFhgVw3ybu2gB5fkk8KLj2idTvzZb8= 2 | github.com/aws/aws-sdk-go v1.44.257/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= 3 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 5 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= 7 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 8 | github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= 9 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 10 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 11 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 12 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 13 | github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= 14 | github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 15 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 16 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 17 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 18 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 19 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 20 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 21 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 22 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 23 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 24 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 25 | golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= 26 | golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= 27 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 28 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 29 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 30 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 31 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 32 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 33 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 34 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 35 | golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 36 | golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= 37 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 38 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 39 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 40 | golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 41 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 42 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 43 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 44 | golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= 45 | golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 46 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 47 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 48 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 49 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 50 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 51 | gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= 52 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 53 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 54 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 55 | -------------------------------------------------------------------------------- /run/core/aws-sdk-go/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # run tests 28 | /mint/run/core/aws-sdk-go/aws-sdk-go 1>>"$output_log_file" 2>"$error_log_file" 29 | -------------------------------------------------------------------------------- /run/core/aws-sdk-java-v2/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2018 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # run tests 28 | cd /mint/run/core/aws-sdk-java-v2/ || exit 1 29 | 30 | java -jar FunctionalTests.jar 1>>"$output_log_file" 2>"$error_log_file" 31 | -------------------------------------------------------------------------------- /run/core/aws-sdk-java/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2018 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # run tests 28 | cd /mint/run/core/aws-sdk-java/ || exit 1 29 | 30 | java -jar FunctionalTests.jar 1>>"$output_log_file" 2>"$error_log_file" 31 | -------------------------------------------------------------------------------- /run/core/aws-sdk-php/README.md: -------------------------------------------------------------------------------- 1 | ## `aws-sdk-php` tests 2 | This directory serves as the location for Mint tests using `aws-sdk-php`. Top level `mint.sh` calls `run.sh` to execute tests. 3 | 4 | ## Adding new tests 5 | New tests is added into `quick-tests.php` as new functions. 6 | 7 | ## Running tests manually 8 | - Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` 9 | - Call `run.sh` with output log file and error log file. for example 10 | ```bash 11 | export MINT_DATA_DIR=~/my-mint-dir 12 | export MINT_MODE=core 13 | export SERVER_ENDPOINT="play.minio.io:9000" 14 | export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" 15 | export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" 16 | export ENABLE_HTTPS=1 17 | export SERVER_REGION=us-east-1 18 | ./run.sh /tmp/output.log /tmp/error.log 19 | ``` 20 | -------------------------------------------------------------------------------- /run/core/aws-sdk-php/composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "require": { 3 | "aws/aws-sdk-php": "^3.30", 4 | "guzzlehttp/psr7": "^1.4" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /run/core/aws-sdk-php/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint, (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # run tests 28 | php ./quick-tests.php 1>>"$output_log_file" 2>"$error_log_file" 29 | -------------------------------------------------------------------------------- /run/core/aws-sdk-ruby/README.md: -------------------------------------------------------------------------------- 1 | ## `aws-sdk-ruby` tests 2 | This directory serves as the location for Mint tests using `aws-sdk-ruby`. Top level `mint.sh` calls `run.sh` to execute tests. 3 | 4 | ## Adding new tests 5 | New tests is added into `aws-stub-test.rb` as new functions. 6 | 7 | ## Running tests manually 8 | - Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` 9 | - Call `run.sh` with output log file and error log file. for example 10 | ```bash 11 | export MINT_DATA_DIR=~/my-mint-dir 12 | export MINT_MODE=core 13 | export SERVER_ENDPOINT="play.minio.io:9000" 14 | export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" 15 | export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" 16 | export ENABLE_HTTPS=1 17 | export SERVER_REGION=us-east-1 18 | ./run.sh /tmp/output.log /tmp/error.log 19 | ``` 20 | -------------------------------------------------------------------------------- /run/core/aws-sdk-ruby/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # run tests 28 | chmod a+x aws-stub-tests.rb 29 | ruby aws-stub-tests.rb 1>>"$output_log_file" 2>"$error_log_file" 30 | -------------------------------------------------------------------------------- /run/core/awscli/README.md: -------------------------------------------------------------------------------- 1 | ## `awscli` tests 2 | This directory serves as the location for Mint tests using `awscli`. Top level `mint.sh` calls `run.sh` to execute tests. 3 | 4 | ## Adding new tests 5 | New tests is added into `test.sh` as new functions. 6 | 7 | ## Running tests manually 8 | - Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` 9 | - Call `run.sh` with output log file and error log file. for example 10 | ```bash 11 | export MINT_DATA_DIR=~/my-mint-dir 12 | export MINT_MODE=core 13 | export SERVER_ENDPOINT="play.minio.io:9000" 14 | export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" 15 | export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" 16 | export ENABLE_HTTPS=1 17 | export SERVER_REGION=us-east-1 18 | ./run.sh /tmp/output.log /tmp/error.log 19 | ``` 20 | -------------------------------------------------------------------------------- /run/core/awscli/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # configure awscli 28 | aws configure set aws_access_key_id "$ACCESS_KEY" 29 | aws configure set aws_secret_access_key "$SECRET_KEY" 30 | aws configure set default.region "$SERVER_REGION" 31 | 32 | # run tests for virtual style if provided 33 | if [ "$ENABLE_VIRTUAL_STYLE" -eq 1 ]; then 34 | # Setup endpoint scheme 35 | endpoint="http://$DOMAIN:$SERVER_PORT" 36 | if [ "$ENABLE_HTTPS" -eq 1 ]; then 37 | endpoint="https://$DOMAIN:$SERVER_PORT" 38 | fi 39 | dnsmasq --address="/$DOMAIN/$SERVER_IP" --user=root 40 | echo -e "nameserver 127.0.0.1\n$(cat /etc/resolv.conf)" >/etc/resolv.conf 41 | aws configure set default.s3.addressing_style virtual 42 | ./test.sh "$endpoint" 1>>"$output_log_file" 2>"$error_log_file" 43 | aws configure set default.s3.addressing_style path 44 | fi 45 | 46 | endpoint="http://$SERVER_ENDPOINT" 47 | if [ "$ENABLE_HTTPS" -eq 1 ]; then 48 | endpoint="https://$SERVER_ENDPOINT" 49 | fi 50 | # run path style tests 51 | ./test.sh "$endpoint" 1>>"$output_log_file" 2>"$error_log_file" 52 | -------------------------------------------------------------------------------- /run/core/healthcheck/go.mod: -------------------------------------------------------------------------------- 1 | module mint.minio.io/healthcheck 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/golang-jwt/jwt/v4 v4.5.2 7 | github.com/sirupsen/logrus v1.9.0 8 | ) 9 | 10 | require golang.org/x/sys v0.5.0 // indirect 11 | -------------------------------------------------------------------------------- /run/core/healthcheck/go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= 5 | github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= 6 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 7 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 8 | github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= 9 | github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 10 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 11 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 12 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 13 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 14 | golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= 15 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 16 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 17 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 18 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 19 | -------------------------------------------------------------------------------- /run/core/healthcheck/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Mint, (C) 2019 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | package main 21 | 22 | import ( 23 | "crypto/tls" 24 | "encoding/json" 25 | "fmt" 26 | "net/http" 27 | "net/url" 28 | "os" 29 | "time" 30 | 31 | jwtgo "github.com/golang-jwt/jwt/v4" 32 | log "github.com/sirupsen/logrus" 33 | ) 34 | 35 | const ( 36 | pass = "PASS" // Indicate that a test passed 37 | fail = "FAIL" // Indicate that a test failed 38 | livenessPath = "/minio/health/live" 39 | readinessPath = "/minio/health/ready" 40 | prometheusPathV2Cluster = "/minio/v2/metrics/cluster" 41 | prometheusPathV2Node = "/minio/v2/metrics/node" 42 | prometheusPathV2Bucket = "/minio/v2/metrics/bucket" 43 | prometheusPathV2Resource = "/minio/v2/metrics/resource" 44 | timeout = time.Duration(30 * time.Second) 45 | ) 46 | 47 | type mintJSONFormatter struct{} 48 | 49 | func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) { 50 | data := make(log.Fields, len(entry.Data)) 51 | for k, v := range entry.Data { 52 | switch v := v.(type) { 53 | case error: 54 | // Otherwise errors are ignored by `encoding/json` 55 | // https://github.com/sirupsen/logrus/issues/137 56 | data[k] = v.Error() 57 | default: 58 | data[k] = v 59 | } 60 | } 61 | 62 | serialized, err := json.Marshal(data) 63 | if err != nil { 64 | return nil, fmt.Errorf("Failed to marshal fields to JSON, %w", err) 65 | } 66 | return append(serialized, '\n'), nil 67 | } 68 | 69 | // log successful test runs 70 | func successLogger(function string, args map[string]interface{}, startTime time.Time) *log.Entry { 71 | // calculate the test case duration 72 | duration := time.Since(startTime) 73 | // log with the fields as per mint 74 | fields := log.Fields{"name": "healthcheck", "function": function, "args": args, "duration": duration.Nanoseconds() / 1000000, "status": pass} 75 | return log.WithFields(fields) 76 | } 77 | 78 | // log failed test runs 79 | func failureLog(function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) *log.Entry { 80 | // calculate the test case duration 81 | duration := time.Since(startTime) 82 | var fields log.Fields 83 | // log with the fields as per mint 84 | if err != nil { 85 | fields = log.Fields{ 86 | "name": "healthcheck", "function": function, "args": args, 87 | "duration": duration.Nanoseconds() / 1000000, "status": fail, "alert": alert, "message": message, "error": err, 88 | } 89 | } else { 90 | fields = log.Fields{ 91 | "name": "healthcheck", "function": function, "args": args, 92 | "duration": duration.Nanoseconds() / 1000000, "status": fail, "alert": alert, "message": message, 93 | } 94 | } 95 | return log.WithFields(fields) 96 | } 97 | 98 | func testLivenessEndpoint(endpoint string) { 99 | startTime := time.Now() 100 | function := "testLivenessEndpoint" 101 | 102 | u, err := url.Parse(fmt.Sprintf("%s%s", endpoint, livenessPath)) 103 | if err != nil { 104 | // Could not parse URL successfully 105 | failureLog(function, nil, startTime, "", "URL Parsing for Healthcheck Liveness handler failed", err).Fatal() 106 | } 107 | 108 | tr := &http.Transport{ 109 | TLSClientConfig: &tls.Config{InsecureSkipVerify: u.Scheme == "https"}, 110 | } 111 | client := &http.Client{Transport: tr, Timeout: timeout} 112 | resp, err := client.Get(u.String()) 113 | if err != nil { 114 | // GET request errored 115 | failureLog(function, nil, startTime, "", "GET request failed", err).Fatal() 116 | } 117 | if resp.StatusCode != http.StatusOK { 118 | // Status not 200 OK 119 | failureLog(function, nil, startTime, "", fmt.Sprintf("GET /minio/health/live returned %s", resp.Status), err).Fatal() 120 | } 121 | 122 | defer resp.Body.Close() 123 | defer successLogger(function, nil, startTime).Info() 124 | } 125 | 126 | func testReadinessEndpoint(endpoint string) { 127 | startTime := time.Now() 128 | function := "testReadinessEndpoint" 129 | 130 | u, err := url.Parse(fmt.Sprintf("%s%s", endpoint, readinessPath)) 131 | if err != nil { 132 | // Could not parse URL successfully 133 | failureLog(function, nil, startTime, "", "URL Parsing for Healthcheck Readiness handler failed", err).Fatal() 134 | } 135 | 136 | tr := &http.Transport{ 137 | TLSClientConfig: &tls.Config{InsecureSkipVerify: u.Scheme == "https"}, 138 | } 139 | client := &http.Client{Transport: tr, Timeout: timeout} 140 | resp, err := client.Get(u.String()) 141 | if err != nil { 142 | // GET request errored 143 | failureLog(function, nil, startTime, "", "GET request to Readiness endpoint failed", err).Fatal() 144 | } 145 | if resp.StatusCode != http.StatusOK { 146 | // Status not 200 OK 147 | failureLog(function, nil, startTime, "", "GET /minio/health/ready returned non OK status", err).Fatal() 148 | } 149 | 150 | defer resp.Body.Close() 151 | defer successLogger(function, nil, startTime).Info() 152 | } 153 | 154 | const ( 155 | defaultPrometheusJWTExpiry = 100 * 365 * 24 * time.Hour 156 | ) 157 | 158 | func testPrometheusEndpointV2(endpoint string, metricsPath string) { 159 | startTime := time.Now() 160 | function := "testPrometheusEndpoint" 161 | 162 | u, err := url.Parse(fmt.Sprintf("%s%s", endpoint, metricsPath)) 163 | if err != nil { 164 | // Could not parse URL successfully 165 | failureLog(function, nil, startTime, "", "URL Parsing for Healthcheck Prometheus handler failed", err).Fatal() 166 | } 167 | 168 | jwt := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, jwtgo.StandardClaims{ 169 | ExpiresAt: time.Now().UTC().Add(defaultPrometheusJWTExpiry).Unix(), 170 | Subject: os.Getenv("ACCESS_KEY"), 171 | Issuer: "prometheus", 172 | }) 173 | 174 | token, err := jwt.SignedString([]byte(os.Getenv("SECRET_KEY"))) 175 | if err != nil { 176 | failureLog(function, nil, startTime, "", "jwt generation failed", err).Fatal() 177 | } 178 | 179 | tr := &http.Transport{ 180 | TLSClientConfig: &tls.Config{InsecureSkipVerify: u.Scheme == "https"}, 181 | } 182 | client := &http.Client{Transport: tr, Timeout: timeout} 183 | 184 | req, err := http.NewRequest(http.MethodGet, u.String(), nil) 185 | if err != nil { 186 | failureLog(function, nil, startTime, "", "Initializing GET request to Prometheus endpoint failed", err).Fatal() 187 | } 188 | req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) 189 | 190 | resp, err := client.Do(req) 191 | if err != nil { 192 | // GET request errored 193 | failureLog(function, nil, startTime, "", "GET request to Prometheus endpoint failed", err).Fatal() 194 | } 195 | 196 | if resp.StatusCode != http.StatusOK { 197 | // Status not 200 OK 198 | failureLog(function, nil, startTime, "", "GET "+endpoint+" returned non OK status", err).Fatal() 199 | } 200 | 201 | defer resp.Body.Close() 202 | defer successLogger(function, nil, startTime).Info() 203 | } 204 | 205 | func testClusterPrometheusEndpointV2(endpoint string) { 206 | testPrometheusEndpointV2(endpoint, prometheusPathV2Cluster) 207 | } 208 | 209 | func testNodePrometheusEndpointV2(endpoint string) { 210 | testPrometheusEndpointV2(endpoint, prometheusPathV2Node) 211 | } 212 | 213 | func testBucketPrometheusEndpointV2(endpoint string) { 214 | testPrometheusEndpointV2(endpoint, prometheusPathV2Bucket) 215 | } 216 | 217 | func testResourcePrometheusEndpointV2(endpoint string) { 218 | testPrometheusEndpointV2(endpoint, prometheusPathV2Resource) 219 | } 220 | 221 | func main() { 222 | endpoint := os.Getenv("SERVER_ENDPOINT") 223 | secure := os.Getenv("ENABLE_HTTPS") 224 | if secure == "1" { 225 | endpoint = "https://" + endpoint 226 | } else { 227 | endpoint = "http://" + endpoint 228 | } 229 | 230 | // Output to stdout instead of the default stderr 231 | log.SetOutput(os.Stdout) 232 | // create custom formatter 233 | mintFormatter := mintJSONFormatter{} 234 | // set custom formatter 235 | log.SetFormatter(&mintFormatter) 236 | // log Info or above -- success cases are Info level, failures are Fatal level 237 | log.SetLevel(log.InfoLevel) 238 | // execute tests 239 | testLivenessEndpoint(endpoint) 240 | testReadinessEndpoint(endpoint) 241 | testClusterPrometheusEndpointV2(endpoint) 242 | testNodePrometheusEndpointV2(endpoint) 243 | testBucketPrometheusEndpointV2(endpoint) 244 | testResourcePrometheusEndpointV2(endpoint) 245 | } 246 | -------------------------------------------------------------------------------- /run/core/healthcheck/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2019 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # run tests 28 | /mint/run/core/healthcheck/healthcheck 1>>"$output_log_file" 2>"$error_log_file" 29 | -------------------------------------------------------------------------------- /run/core/mc/README.md: -------------------------------------------------------------------------------- 1 | ## `mc` tests 2 | This directory serves as the location for Mint tests using `mc`. Top level `mint.sh` calls `run.sh` to execute tests. 3 | 4 | ## Adding new tests 5 | New tests is added into `test.sh` as new functions. 6 | 7 | ## Running tests manually 8 | - Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` 9 | - Call `run.sh` with output log file and error log file. for example 10 | ```bash 11 | export MINT_DATA_DIR=~/my-mint-dir 12 | export MINT_MODE=core 13 | export SERVER_ENDPOINT="play.minio.io:9000" 14 | export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" 15 | export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" 16 | export ENABLE_HTTPS=1 17 | export SERVER_REGION=us-east-1 18 | ./run.sh /tmp/output.log /tmp/error.log 19 | ``` 20 | -------------------------------------------------------------------------------- /run/core/mc/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Minio Cloud Storage, (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | ./functional-tests.sh 1>>"$output_log_file" 2>"$error_log_file" 28 | -------------------------------------------------------------------------------- /run/core/minio-go/README.md: -------------------------------------------------------------------------------- 1 | ## `minio-go` tests 2 | This directory serves as the location for Mint tests using `minio-go`. Top level `mint.sh` calls `run.sh` to execute tests. 3 | 4 | ## Adding new tests 5 | New tests are added in functional tests of minio-go. Please check https://github.com/minio/minio-go 6 | 7 | ## Running tests manually 8 | - Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION`, `ENABLE_HTTPS` and `RUN_ON_FAIL` 9 | - Call `run.sh` with output log file and error log file. for example 10 | ```bash 11 | export MINT_DATA_DIR=~/my-mint-dir 12 | export MINT_MODE=core 13 | export SERVER_ENDPOINT="play.minio.io:9000" 14 | export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" 15 | export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" 16 | export ENABLE_HTTPS=1 17 | export SERVER_REGION=us-east-1 18 | export RUN_ON_FAIL=1 19 | ./run.sh /tmp/output.log /tmp/error.log 20 | ``` 21 | -------------------------------------------------------------------------------- /run/core/minio-go/go.mod: -------------------------------------------------------------------------------- 1 | module mint.minio.io/minio-go 2 | 3 | go 1.19 4 | -------------------------------------------------------------------------------- /run/core/minio-go/go.sum: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/minio/mint/befedef1f35389666df0885fe2157118c6f425c3/run/core/minio-go/go.sum -------------------------------------------------------------------------------- /run/core/minio-go/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # run tests 28 | /mint/run/core/minio-go/minio-go 1>>"$output_log_file" 2>"$error_log_file" 29 | -------------------------------------------------------------------------------- /run/core/minio-java/README.md: -------------------------------------------------------------------------------- 1 | ## `minio-java` tests 2 | This directory serves as the location for Mint tests using `minio-java`. Top level `mint.sh` calls `run.sh` to execute tests. 3 | 4 | ## Adding new tests 5 | New tests is added in functional tests of minio-java. Please check https://github.com/minio/minio-java 6 | 7 | ## Running tests manually 8 | - Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION`, `ENABLE_HTTPS` and `RUN_ON_FAIL` 9 | - Call `run.sh` with output log file and error log file. for example 10 | ```bash 11 | export MINT_DATA_DIR=~/my-mint-dir 12 | export MINT_MODE=core 13 | export SERVER_ENDPOINT="play.minio.io:9000" 14 | export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" 15 | export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" 16 | export ENABLE_HTTPS=1 17 | export SERVER_REGION=us-east-1 18 | export RUN_ON_FAIL=1 19 | ./run.sh /tmp/output.log /tmp/error.log 20 | ``` 21 | -------------------------------------------------------------------------------- /run/core/minio-java/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # run tests 28 | endpoint="http://$SERVER_ENDPOINT" 29 | if [ "$ENABLE_HTTPS" -eq 1 ]; then 30 | endpoint="https://$SERVER_ENDPOINT" 31 | fi 32 | 33 | java -Xmx4096m -Xms256m -cp "/mint/run/core/minio-java/*:." FunctionalTest \ 34 | "$endpoint" "$ACCESS_KEY" "$SECRET_KEY" "$SERVER_REGION" 1>>"$output_log_file" 2>"$error_log_file" 35 | -------------------------------------------------------------------------------- /run/core/minio-js/.mocharc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | spec: 'test/**/*.js', 3 | exit: true, 4 | reporter: 'spec', 5 | ui: 'bdd', 6 | require: ['dotenv/config', 'source-map-support/register', './babel-register.js'], 7 | } -------------------------------------------------------------------------------- /run/core/minio-js/README.md: -------------------------------------------------------------------------------- 1 | ## `minio-js` tests 2 | This directory serves as the location for Mint tests using `minio-js`. Top level `mint.sh` calls `run.sh` to execute tests. 3 | 4 | ## Adding new tests 5 | New tests is added in functional tests of minio-js. Please check https://github.com/minio/minio-js 6 | 7 | ## Running tests manually 8 | - Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` 9 | - Call `run.sh` with output log file and error log file. for example 10 | ```bash 11 | export MINT_DATA_DIR=~/my-mint-dir 12 | export MINT_MODE=core 13 | export SERVER_ENDPOINT="play.minio.io:9000" 14 | export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" 15 | export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" 16 | export ENABLE_HTTPS=1 17 | export SERVER_REGION=us-east-1 18 | ./run.sh /tmp/output.log /tmp/error.log 19 | ``` 20 | -------------------------------------------------------------------------------- /run/core/minio-js/babel-register.js: -------------------------------------------------------------------------------- 1 | // fix babel register doesn't transform TypeScript 2 | // 3 | // https://github.com/babel/babel/issues/8962#issuecomment-443135379 4 | 5 | // eslint-disable-next-line @typescript-eslint/no-var-requires,import/no-commonjs 6 | const register = require('@babel/register') 7 | 8 | register({ 9 | extensions: ['.ts', '.js'], 10 | assumptions: { 11 | constantSuper: true, 12 | noIncompleteNsImportDetection: true, 13 | constantReexports: true, 14 | }, 15 | plugins: [ 16 | [ 17 | '@babel/plugin-transform-modules-commonjs', 18 | { 19 | importInterop: 'node', 20 | }, 21 | ], 22 | '@upleveled/remove-node-prefix', // lower version of node (<14) doesn't support require('node:fs') 23 | ], 24 | presets: [ 25 | ['@babel/preset-typescript', { allExtensions: true }], 26 | ['@babel/preset-env', { targets: { node: 'current' }, modules: 'cjs' }], 27 | ], 28 | }) 29 | -------------------------------------------------------------------------------- /run/core/minio-js/minioreporter.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Reporter for JSON formatted logging, (C) 2017 Minio, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | var mocha = require('mocha'); 18 | module.exports = minioreporter; 19 | 20 | function minioreporter(runner) { 21 | mocha.reporters.Base.call(this, runner); 22 | var self = this; 23 | 24 | runner.on('pass', function (test) { 25 | GenerateJsonEntry(test) 26 | }); 27 | 28 | runner.on('fail', function (test, err) { 29 | GenerateJsonEntry(test, err) 30 | }); 31 | 32 | } 33 | 34 | /** 35 | * Convert test result into a JSON object and print on the console. 36 | * 37 | * @api private 38 | * @param test, err 39 | */ 40 | 41 | function GenerateJsonEntry (test, err) { 42 | var res = test.title.split("_") 43 | var jsonEntry = {}; 44 | 45 | jsonEntry.name = "minio-js" 46 | 47 | if (res.length > 0 && res[0].length) { 48 | jsonEntry.function = res[0] 49 | } 50 | 51 | if (res.length > 1 && res[1].length) { 52 | jsonEntry.args = res[1] 53 | } 54 | 55 | jsonEntry.duration = test.duration 56 | 57 | if (res.length > 2 && res[2].length) { 58 | jsonEntry.alert = res[2] 59 | } 60 | 61 | if (err != null ) { 62 | jsonEntry.status = "FAIL" 63 | jsonEntry.error = err.stack.replace(/\n/g, " ").replace(/ +(?= )/g,'') 64 | } else { 65 | jsonEntry.status = "PASS" 66 | } 67 | 68 | process.stdout.write(JSON.stringify(jsonEntry) + "\n") 69 | } 70 | -------------------------------------------------------------------------------- /run/core/minio-js/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bin", 3 | "version": "1.0.0", 4 | "keywords": ["mint minio js"], 5 | "author": "", 6 | "license": "ISC", 7 | "description": "", 8 | "dependencies": { 9 | "async": "^3.2.4", 10 | "block-stream2": "^2.1.0", 11 | "browser-or-node": "^2.1.1", 12 | "buffer-crc32": "^0.2.13", 13 | "fast-xml-parser": "^4.2.2", 14 | "ipaddr.js": "^2.0.1", 15 | "json-stream": "^1.0.0", 16 | "lodash": "^4.17.21", 17 | "mime-types": "^2.1.35", 18 | "query-string": "^7.1.3", 19 | "through2": "^4.0.2", 20 | "web-encoding": "^1.1.5", 21 | "xml": "^1.0.1", 22 | "xml2js": "^0.5.0" 23 | }, 24 | "devDependencies": { 25 | "@babel/core": "^7.21.8", 26 | "@babel/plugin-transform-modules-commonjs": "^7.21.5", 27 | "@babel/preset-env": "^7.21.5", 28 | "@babel/preset-typescript": "^7.21.5", 29 | "@babel/register": "^7.21.0", 30 | "@nodelib/fs.walk": "^1.2.8", 31 | "@types/async": "^3.2.20", 32 | "@types/lodash": "^4.14.194", 33 | "@types/mime-types": "^2.1.1", 34 | "@types/node": "^20.1.0", 35 | "@types/xml": "^1.0.8", 36 | "@types/xml2js": "^0.4.11", 37 | "@typescript-eslint/eslint-plugin": "^5.59.2", 38 | "@typescript-eslint/parser": "^5.59.2", 39 | "@upleveled/babel-plugin-remove-node-prefix": "^1.0.5", 40 | "babel-plugin-replace-import-extension": "^1.1.3", 41 | "babel-plugin-transform-replace-expressions": "^0.2.0", 42 | "chai": "^4.3.7", 43 | "dotenv": "^16.0.3", 44 | "eslint": "^8.40.0", 45 | "eslint-config-prettier": "^8.8.0", 46 | "eslint-import-resolver-typescript": "^3.5.5", 47 | "eslint-plugin-import": "^2.27.5", 48 | "eslint-plugin-simple-import-sort": "^10.0.0", 49 | "eslint-plugin-unicorn": "^47.0.0", 50 | "eslint-plugin-unused-imports": "^2.0.0", 51 | "husky": "^8.0.3", 52 | "lint-staged": "^13.2.2", 53 | "mocha": "^10.2.0", 54 | "mocha-steps": "^1.3.0", 55 | "nock": "^13.3.1", 56 | "prettier": "^2.8.8", 57 | "source-map-support": "^0.5.21", 58 | "split-file": "^2.3.0", 59 | "superagent": "^8.0.1", 60 | "typescript": "^5.0.4", 61 | "uuid": "^9.0.0" 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /run/core/minio-js/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Minio Cloud Storage, (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # Prepare test runner project 28 | cp -R /mint/test-run/minio-js/ ./minio-js 29 | npm install --quiet &>/dev/null 30 | 31 | # Run tests 32 | node ./node_modules/mocha/bin/mocha.js "./minio-js/tests/functional/functional-tests.js" -R minioreporter >>"$output_log_file" 2>"$error_log_file" 33 | -------------------------------------------------------------------------------- /run/core/minio-py/README.md: -------------------------------------------------------------------------------- 1 | ## `minio-py` tests 2 | This directory serves as the location for Mint tests using `minio-py`. Top level `mint.sh` calls `run.sh` to execute tests. 3 | 4 | ## Adding new tests 5 | New tests is added in functional tests of minio-py. Please check https://github.com/minio/minio-py 6 | 7 | ## Running tests manually 8 | - Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` 9 | - Call `run.sh` with output log file and error log file. for example 10 | ```bash 11 | export MINT_DATA_DIR=~/my-mint-dir 12 | export MINT_MODE=core 13 | export SERVER_ENDPOINT="play.minio.io:9000" 14 | export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" 15 | export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" 16 | export ENABLE_HTTPS=1 17 | export SERVER_REGION=us-east-1 18 | ./run.sh /tmp/output.log /tmp/error.log 19 | ``` 20 | -------------------------------------------------------------------------------- /run/core/minio-py/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # run tests 28 | python "/mint/run/core/minio-py/tests.py" 1>>"$output_log_file" 2>"$error_log_file" 29 | -------------------------------------------------------------------------------- /run/core/s3cmd/.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.log 3 | -------------------------------------------------------------------------------- /run/core/s3cmd/README.md: -------------------------------------------------------------------------------- 1 | ## `s3cmd` tests 2 | This directory serves as the location for Mint tests using `s3cmd`. Top level `mint.sh` calls `run.sh` to execute tests. 3 | 4 | ## Adding new tests 5 | New tests is added into `test.sh` as new functions. 6 | 7 | ## Running tests manually 8 | - Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` 9 | - Call `run.sh` with output log file and error log file. for example 10 | ```bash 11 | export MINT_DATA_DIR=~/my-mint-dir 12 | export MINT_MODE=core 13 | export SERVER_ENDPOINT="play.minio.io:9000" 14 | export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" 15 | export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" 16 | export ENABLE_HTTPS=1 17 | export SERVER_REGION=us-east-1 18 | ./run.sh /tmp/output.log /tmp/error.log 19 | ``` 20 | -------------------------------------------------------------------------------- /run/core/s3cmd/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # run tests 28 | ./test.sh 1>>"$output_log_file" 2>"$error_log_file" 29 | -------------------------------------------------------------------------------- /run/core/s3select/README.md: -------------------------------------------------------------------------------- 1 | 2 | ## `s3select` tests 3 | This directory serves as the location for Mint tests for s3select features. Top level `mint.sh` calls `run.sh` to execute tests. 4 | 5 | ## Adding new tests 6 | New tests are added into `s3select/tests.py` as new functions. 7 | 8 | ## Running tests manually 9 | - Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` 10 | - Call `run.sh` with output log file and error log file. for example 11 | 12 | ```bash 13 | export MINT_DATA_DIR=~/my-mint-dir 14 | export MINT_MODE=core 15 | export SERVER_ENDPOINT="play.min.io" 16 | export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" 17 | export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" 18 | export ENABLE_HTTPS=1 19 | export SERVER_REGION=us-east-1 20 | ./run.sh /tmp/output.log /tmp/error.log 21 | ``` 22 | -------------------------------------------------------------------------------- /run/core/s3select/csv.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # MinIO Python Library for Amazon S3 Compatible Cloud Storage, 4 | # (C) 2015-2020 MinIO, Inc. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | import io 19 | import os 20 | 21 | from minio import Minio 22 | from minio.select import (COMPRESSION_TYPE_NONE, FILE_HEADER_INFO_NONE, 23 | JSON_TYPE_DOCUMENT, QUOTE_FIELDS_ALWAYS, 24 | QUOTE_FIELDS_ASNEEDED, CSVInputSerialization, 25 | CSVOutputSerialization, JSONInputSerialization, 26 | JSONOutputSerialization, SelectRequest) 27 | 28 | from utils import * 29 | 30 | 31 | def test_sql_api(test_name, client, bucket_name, input_data, sql_opts, expected_output): 32 | """ Test if the passed SQL request has the output equal to the passed execpted one""" 33 | object_name = generate_object_name() 34 | got_output = b'' 35 | try: 36 | bytes_content = io.BytesIO(input_data) 37 | client.put_object(bucket_name, object_name, 38 | io.BytesIO(input_data), len(input_data)) 39 | data = client.select_object_content(bucket_name, object_name, sql_opts) 40 | # Get the records 41 | records = io.BytesIO() 42 | for d in data.stream(10*1024): 43 | records.write(d) 44 | got_output = records.getvalue() 45 | except Exception as select_err: 46 | if not isinstance(expected_output, Exception): 47 | raise ValueError( 48 | 'Test {} unexpectedly failed with: {}'.format(test_name, select_err)) 49 | else: 50 | if isinstance(expected_output, Exception): 51 | raise ValueError( 52 | 'Test {}: expected an exception, got {}'.format(test_name, got_output)) 53 | if got_output != expected_output: 54 | raise ValueError('Test {}: data mismatch. Expected : {}, Received {}'.format( 55 | test_name, expected_output, got_output)) 56 | finally: 57 | client.remove_object(bucket_name, object_name) 58 | 59 | 60 | def test_csv_input_custom_quote_char(client, log_output): 61 | # Get a unique bucket_name and object_name 62 | log_output.args['bucket_name'] = bucket_name = generate_bucket_name() 63 | 64 | tests = [ 65 | # Invalid quote character, should fail 66 | ('""', '"', b'col1,col2,col3\n', Exception()), 67 | # UTF-8 quote character 68 | ('ع', '"', 'عcol1ع,عcol2ع,عcol3ع\n'.encode(), 69 | b'{"_1":"col1","_2":"col2","_3":"col3"}\n'), 70 | # Only one field is quoted 71 | ('"', '"', b'"col1",col2,col3\n', 72 | b'{"_1":"col1","_2":"col2","_3":"col3"}\n'), 73 | ('"', '"', b'"col1,col2,col3"\n', b'{"_1":"col1,col2,col3"}\n'), 74 | ('\'', '"', b'"col1",col2,col3\n', 75 | b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'), 76 | ('', '"', b'"col1",col2,col3\n', 77 | b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'), 78 | ('', '"', b'"col1",col2,col3\n', 79 | b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'), 80 | ('', '"', b'"col1","col2","col3"\n', 81 | b'{"_1":"\\"col1\\"","_2":"\\"col2\\"","_3":"\\"col3\\""}\n'), 82 | ('"', '"', b'""""""\n', b'{"_1":"\\"\\""}\n'), 83 | ('"', '"', b'A",B\n', b'{"_1":"A\\"","_2":"B"}\n'), 84 | ('"', '"', b'A"",B\n', b'{"_1":"A\\"\\"","_2":"B"}\n'), 85 | ('"', '\\', b'A\\B,C\n', b'{"_1":"A\\\\B","_2":"C"}\n'), 86 | ('"', '"', b'"A""B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'), 87 | ('"', '\\', b'"A\\B","CD"\n', b'{"_1":"AB","_2":"CD"}\n'), 88 | ('"', '\\', b'"A\\,","CD"\n', b'{"_1":"A,","_2":"CD"}\n'), 89 | ('"', '\\', b'"A\\"B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'), 90 | ('"', '\\', b'"A\\""\n', b'{"_1":"A\\""}\n'), 91 | ('"', '\\', b'"A\\"\\"B"\n', b'{"_1":"A\\"\\"B"}\n'), 92 | ('"', '\\', b'"A\\"","\\"B"\n', b'{"_1":"A\\"","_2":"\\"B"}\n'), 93 | ] 94 | 95 | client.make_bucket(bucket_name) 96 | 97 | try: 98 | for idx, (quote_char, escape_char, data, expected_output) in enumerate(tests): 99 | sql_opts = SelectRequest( 100 | "select * from s3object", 101 | CSVInputSerialization( 102 | compression_type=COMPRESSION_TYPE_NONE, 103 | file_header_info=FILE_HEADER_INFO_NONE, 104 | record_delimiter="\n", 105 | field_delimiter=",", 106 | quote_character=quote_char, 107 | quote_escape_character=escape_char, 108 | comments="#", 109 | allow_quoted_record_delimiter="FALSE", 110 | ), 111 | JSONOutputSerialization( 112 | record_delimiter="\n", 113 | ), 114 | request_progress=False, 115 | ) 116 | 117 | test_sql_api(f'test_{idx}', client, bucket_name, 118 | data, sql_opts, expected_output) 119 | finally: 120 | client.remove_bucket(bucket_name) 121 | 122 | # Test passes 123 | print(log_output.json_report()) 124 | 125 | 126 | def test_csv_output_custom_quote_char(client, log_output): 127 | # Get a unique bucket_name and object_name 128 | log_output.args['bucket_name'] = bucket_name = generate_bucket_name() 129 | 130 | tests = [ 131 | # UTF-8 quote character 132 | ("''", "''", b'col1,col2,col3\n', Exception()), 133 | ("'", "'", b'col1,col2,col3\n', b"'col1','col2','col3'\n"), 134 | ("", '"', b'col1,col2,col3\n', b'\x00col1\x00,\x00col2\x00,\x00col3\x00\n'), 135 | ('"', '"', b'col1,col2,col3\n', b'"col1","col2","col3"\n'), 136 | ('"', '"', b'col"1,col2,col3\n', b'"col""1","col2","col3"\n'), 137 | ('"', '"', b'""""\n', b'""""\n'), 138 | ('"', '"', b'\n', b''), 139 | ("'", "\\", b'col1,col2,col3\n', b"'col1','col2','col3'\n"), 140 | ("'", "\\", b'col""1,col2,col3\n', b"'col\"\"1','col2','col3'\n"), 141 | ("'", "\\", b'col\'1,col2,col3\n', b"'col\\'1','col2','col3'\n"), 142 | ("'", "\\", b'"col\'1","col2","col3"\n', b"'col\\'1','col2','col3'\n"), 143 | ("'", "\\", b'col\'\n', b"'col\\''\n"), 144 | # Two consecutive escaped quotes 145 | ("'", "\\", b'"a"""""\n', b"'a\"\"'\n"), 146 | ] 147 | 148 | client.make_bucket(bucket_name) 149 | 150 | try: 151 | for idx, (quote_char, escape_char, input_data, expected_output) in enumerate(tests): 152 | sql_opts = SelectRequest( 153 | "select * from s3object", 154 | CSVInputSerialization( 155 | compression_type=COMPRESSION_TYPE_NONE, 156 | file_header_info=FILE_HEADER_INFO_NONE, 157 | record_delimiter="\n", 158 | field_delimiter=",", 159 | quote_character='"', 160 | quote_escape_character='"', 161 | comments="#", 162 | allow_quoted_record_delimiter="FALSE", 163 | ), 164 | CSVOutputSerialization( 165 | quote_fields=QUOTE_FIELDS_ALWAYS, 166 | record_delimiter="\n", 167 | field_delimiter=",", 168 | quote_character=quote_char, 169 | quote_escape_character=escape_char, 170 | ), 171 | request_progress=False, 172 | ) 173 | 174 | test_sql_api(f'test_{idx}', client, bucket_name, 175 | input_data, sql_opts, expected_output) 176 | finally: 177 | client.remove_bucket(bucket_name) 178 | 179 | # Test passes 180 | print(log_output.json_report()) 181 | -------------------------------------------------------------------------------- /run/core/s3select/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2020 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # run path style tests 28 | python "./tests.py" 1>>"$output_log_file" 2>"$error_log_file" 29 | -------------------------------------------------------------------------------- /run/core/s3select/tests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # MinIO Python Library for Amazon S3 Compatible Cloud Storage, 4 | # (C) 2015-2020 MinIO, Inc. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | import os 19 | import sys 20 | from csv import (test_csv_input_custom_quote_char, 21 | test_csv_output_custom_quote_char) 22 | 23 | from minio import Minio 24 | 25 | from sql_ops import (test_sql_datatypes, test_sql_functions_agg_cond_conv, 26 | test_sql_functions_date, test_sql_functions_string, 27 | test_sql_operators, test_sql_operators_precedence, 28 | test_sql_select, test_sql_select_csv_no_header, 29 | test_sql_select_json) 30 | from utils import LogOutput 31 | 32 | 33 | def main(): 34 | """ 35 | Functional testing for S3 select. 36 | """ 37 | 38 | try: 39 | access_key = os.getenv('ACCESS_KEY', 'Q3AM3UQ867SPQQA43P2F') 40 | secret_key = os.getenv('SECRET_KEY', 41 | 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG') 42 | server_endpoint = os.getenv('SERVER_ENDPOINT', 'play.min.io') 43 | secure = os.getenv('ENABLE_HTTPS', '1') == '1' 44 | if server_endpoint == 'play.min.io': 45 | access_key = 'Q3AM3UQ867SPQQA43P2F' 46 | secret_key = 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG' 47 | secure = True 48 | 49 | client = Minio(server_endpoint, access_key, secret_key, secure=secure) 50 | 51 | log_output = LogOutput(client.select_object_content, 52 | 'test_csv_input_quote_char') 53 | test_csv_input_custom_quote_char(client, log_output) 54 | 55 | log_output = LogOutput(client.select_object_content, 56 | 'test_csv_output_quote_char') 57 | test_csv_output_custom_quote_char(client, log_output) 58 | 59 | log_output = LogOutput( 60 | client.select_object_content, 'test_sql_operators') 61 | test_sql_operators(client, log_output) 62 | 63 | log_output = LogOutput(client.select_object_content, 64 | 'test_sql_operators_precedence') 65 | test_sql_operators_precedence(client, log_output) 66 | 67 | log_output = LogOutput(client.select_object_content, 68 | 'test_sql_functions_agg_cond_conv') 69 | test_sql_functions_agg_cond_conv(client, log_output) 70 | 71 | log_output = LogOutput( 72 | client.select_object_content, 'test_sql_functions_date') 73 | test_sql_functions_date(client, log_output) 74 | 75 | log_output = LogOutput(client.select_object_content, 76 | 'test_sql_functions_string') 77 | test_sql_functions_string(client, log_output) 78 | 79 | log_output = LogOutput( 80 | client.select_object_content, 'test_sql_datatypes') 81 | test_sql_datatypes(client, log_output) 82 | 83 | log_output = LogOutput(client.select_object_content, 'test_sql_select') 84 | test_sql_select(client, log_output) 85 | 86 | log_output = LogOutput( 87 | client.select_object_content, 'test_sql_select_json') 88 | test_sql_select_json(client, log_output) 89 | 90 | log_output = LogOutput( 91 | client.select_object_content, 'test_sql_select_csv') 92 | test_sql_select_csv_no_header(client, log_output) 93 | 94 | except Exception as err: 95 | print(log_output.json_report(err)) 96 | sys.exit(1) 97 | 98 | 99 | if __name__ == "__main__": 100 | # Execute only if run as a script 101 | main() 102 | -------------------------------------------------------------------------------- /run/core/s3select/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # MinIO Python Library for Amazon S3 Compatible Cloud Storage, 4 | # (C) 2015-2020 MinIO, Inc. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | import inspect 19 | import json 20 | import time 21 | import traceback 22 | import uuid 23 | 24 | 25 | class LogOutput(object): 26 | """ 27 | LogOutput is the class for log output. It is required standard for all 28 | SDK tests controlled by mint. 29 | Here are its attributes: 30 | 'name': name of the SDK under test, e.g. 's3select' 31 | 'function': name of the method/api under test with its signature 32 | The following python code can be used to 33 | pull args information of a and to 34 | put together with the method name: 35 | .__name__+'('+', '.join(args_list)+')' 36 | e.g. 'remove_object(bucket_name, object_name)' 37 | 'args': method/api arguments with their values, in 38 | dictionary form: {'arg1': val1, 'arg2': val2, ...} 39 | 'duration': duration of the whole test in milliseconds, 40 | defaults to 0 41 | 'alert': any extra information user is needed to be alerted about, 42 | like whether this is a Blocker/Gateway/Server related 43 | issue, etc., defaults to None 44 | 'message': descriptive error message, defaults to None 45 | 'error': stack-trace/exception message(only in case of failure), 46 | actual low level exception/error thrown by the program, 47 | defaults to None 48 | 'status': exit status, possible values are 'PASS', 'FAIL', 'NA', 49 | defaults to 'PASS' 50 | """ 51 | 52 | PASS = 'PASS' 53 | FAIL = 'FAIL' 54 | NA = 'NA' 55 | 56 | def __init__(self, meth, test_name): 57 | self.__args_list = inspect.getfullargspec(meth).args[1:] 58 | self.__name = 's3select:'+test_name 59 | self.__function = meth.__name__+'('+', '.join(self.__args_list)+')' 60 | self.__args = {} 61 | self.__duration = 0 62 | self.__alert = '' 63 | self.__message = None 64 | self.__error = None 65 | self.__status = self.PASS 66 | self.__start_time = time.time() 67 | 68 | @property 69 | def name(self): return self.__name 70 | 71 | @property 72 | def function(self): return self.__function 73 | 74 | @property 75 | def args(self): return self.__args 76 | 77 | @name.setter 78 | def name(self, val): self.__name = val 79 | 80 | @function.setter 81 | def function(self, val): self.__function = val 82 | 83 | @args.setter 84 | def args(self, val): self.__args = val 85 | 86 | def json_report(self, err_msg='', alert='', status=''): 87 | self.__args = {k: v for k, v in self.__args.items() if v and v != ''} 88 | entry = {'name': self.__name, 89 | 'function': self.__function, 90 | 'args': self.__args, 91 | 'duration': int(round((time.time() - self.__start_time)*1000)), 92 | 'alert': str(alert), 93 | 'message': str(err_msg), 94 | 'error': traceback.format_exc() if err_msg and err_msg != '' else '', 95 | 'status': status if status and status != '' else 96 | self.FAIL if err_msg and err_msg != '' else self.PASS 97 | } 98 | return json.dumps({k: v for k, v in entry.items() if v and v != ''}) 99 | 100 | 101 | def generate_bucket_name(): 102 | return "s3select-test-" + str(uuid.uuid4()) 103 | 104 | 105 | def generate_object_name(): 106 | return str(uuid.uuid4()) 107 | -------------------------------------------------------------------------------- /run/core/versioning/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2021 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # run tests 28 | /mint/run/core/versioning/tests 1>>"$output_log_file" 2>"$error_log_file" 29 | -------------------------------------------------------------------------------- /source.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Minio Cloud Storage, (C) 2024 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | export MINT_RUN_CORE_DIR="$MINT_ROOT_DIR/run/core" 19 | export MINT_RUN_BUILD_DIR="$MINT_ROOT_DIR/build" 20 | export APT="apt --quiet --yes" 21 | export WGET="wget --quiet --no-check-certificate" 22 | export WGET="wget --quiet --no-check-certificate" 23 | 24 | ## Software versions 25 | export GO_VERSION="1.21.9" 26 | export GRADLE_VERSION="8.5" 27 | export GRADLE_INSTALL_PATH="/opt/gradle" 28 | export GO_INSTALL_PATH="/usr/local" 29 | 30 | export PATH=${GO_INSTALL_PATH}/bin:$PATH 31 | export PATH=${GRADLE_INSTALL_PATH}/gradle-${GRADLE_VERSION}/bin:$PATH 32 | --------------------------------------------------------------------------------