├── run └── core │ ├── minio-go │ ├── go.sum │ ├── go.mod │ ├── README.md │ ├── run.sh │ └── versioning_test.go │ ├── s3cmd │ ├── .gitignore │ ├── README.md │ ├── run.sh │ └── test.sh │ ├── aws-sdk-php │ ├── composer.json │ ├── README.md │ ├── run.sh │ └── quick-tests.php │ ├── minio-js │ ├── .mocharc.js │ ├── README.md │ ├── babel-register.js │ ├── run.sh │ ├── minioreporter.js │ └── package.json │ ├── healthcheck │ ├── go.mod │ ├── run.sh │ ├── go.sum │ └── main.go │ ├── mc │ ├── README.md │ └── run.sh │ ├── awscli │ ├── README.md │ └── run.sh │ ├── aws-sdk-ruby │ ├── README.md │ ├── run.sh │ └── aws-stub-tests.rb │ ├── s3select │ ├── README.md │ ├── run.sh │ ├── tests.py │ ├── utils.py │ ├── csv.py │ └── sql_ops.py │ ├── minio-py │ ├── README.md │ └── run.sh │ ├── minio-java │ ├── README.md │ └── run.sh │ ├── .minio-dotnet │ └── run.sh │ ├── versioning │ └── run.sh │ ├── aws-sdk-go-v2 │ ├── run.sh │ ├── go.mod │ └── go.sum │ └── aws-sdk-java-v2 │ └── run.sh ├── .dockerignore ├── remove-packages.list ├── install-packages.list ├── .gitignore ├── Dockerfile ├── .github └── workflows │ ├── shellsheck.yml │ ├── dependency-review.yml │ └── codeql-analysis.yml ├── entrypoint.sh ├── postinstall.sh ├── release.sh ├── source.sh ├── create-data-files.sh ├── preinstall.sh ├── mint.sh ├── README.md └── LICENSE /run/core/minio-go/go.sum: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /run/core/s3cmd/.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.log 3 | -------------------------------------------------------------------------------- /run/core/minio-go/go.mod: -------------------------------------------------------------------------------- 1 | module mint.minio.io/minio-go 2 | 3 | go 1.25 4 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | *.patch 3 | run/core/minio-go/main.go 4 | run/core/minio-go/minio-go 5 | -------------------------------------------------------------------------------- /remove-packages.list: -------------------------------------------------------------------------------- 1 | wget 2 | git 3 | python3-pip 4 | ruby-dev 5 | ruby-bundler 6 | openjdk-21-jdk 7 | ant 8 | dotnet-sdk-8.0 9 | -------------------------------------------------------------------------------- /run/core/aws-sdk-php/composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "require": { 3 | "aws/aws-sdk-php": "^3.359", 4 | "guzzlehttp/psr7": "^2.4.5" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /run/core/minio-js/.mocharc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | spec: 'test/**/*.js', 3 | exit: true, 4 | reporter: 'spec', 5 | ui: 'bdd', 6 | require: ['dotenv/config', 'source-map-support/register', './babel-register.js'], 7 | } -------------------------------------------------------------------------------- /run/core/healthcheck/go.mod: -------------------------------------------------------------------------------- 1 | module mint.minio.io/healthcheck 2 | 3 | go 1.25 4 | 5 | require ( 6 | github.com/golang-jwt/jwt/v4 v4.5.2 7 | github.com/sirupsen/logrus v1.9.3 8 | ) 9 | 10 | require golang.org/x/sys v0.5.0 // indirect 11 | -------------------------------------------------------------------------------- /install-packages.list: -------------------------------------------------------------------------------- 1 | git 2 | python3-pip 3 | nodejs 4 | openjdk-21-jdk 5 | openjdk-21-jre-headless 6 | dirmngr 7 | apt-transport-https 8 | dotnet-sdk-8.0 9 | ca-certificates-mono 10 | libunwind8 11 | ruby 12 | ruby-dev 13 | ruby-bundler 14 | php 15 | php-curl 16 | php-xml 17 | ant 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.test 2 | *.jar 3 | src/* 4 | temp 5 | __pycache__/ 6 | log/* 7 | minio.test 8 | bin/* 9 | node_modules 10 | # exception to the rule 11 | !log/.gitkeep 12 | !bin/.gitkeep 13 | *.class 14 | *~ 15 | run/core/minio-dotnet/bin/* 16 | run/core/minio-dotnet/obj/* 17 | run/core/minio-dotnet/out/* 18 | run/core/minio-go/main.go 19 | run/core/healthcheck/healthcheck 20 | .idea/ 21 | .phive/ 22 | .phplint.cache/ 23 | build/versioning/tests 24 | tools/ -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:24.04 2 | 3 | ENV DEBIAN_FRONTEND noninteractive 4 | ENV LANG C.UTF-8 5 | ENV GOROOT /usr/local/go 6 | ENV GOPATH /usr/local/gopath 7 | ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH 8 | ENV MINT_ROOT_DIR /mint 9 | 10 | RUN apt-get --yes update && apt-get --yes upgrade && \ 11 | apt-get --yes --quiet install wget jq curl git dnsmasq 12 | 13 | COPY . /mint 14 | 15 | WORKDIR /mint 16 | 17 | RUN /mint/create-data-files.sh 18 | RUN /mint/preinstall.sh 19 | RUN /mint/release.sh 20 | 21 | ENTRYPOINT ["/mint/entrypoint.sh"] 22 | -------------------------------------------------------------------------------- /.github/workflows/shellsheck.yml: -------------------------------------------------------------------------------- 1 | name: 'Trigger: Shell scripts formatting action' 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | sh-checker: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v1 13 | - name: Run the sh-checker 14 | uses: luizm/action-sh-checker@master 15 | env: 16 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 17 | SHFMT_OPTS: -s # arguments to shfmt. 18 | with: 19 | sh_checker_comment: true 20 | sh_checker_exclude: gradlew 21 | -------------------------------------------------------------------------------- /run/core/mc/README.md: -------------------------------------------------------------------------------- 1 | ## `mc` tests 2 | This directory serves as the location for Mint tests using `mc`. Top level `mint.sh` calls `run.sh` to execute tests. 3 | 4 | ## Adding new tests 5 | New tests is added into `test.sh` as new functions. 6 | 7 | ## Running tests manually 8 | - Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` 9 | - Call `run.sh` with output log file and error log file. for example 10 | ```bash 11 | export MINT_DATA_DIR=~/my-mint-dir 12 | export MINT_MODE=core 13 | export SERVER_ENDPOINT="play.minio.io:9000" 14 | export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" 15 | export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" 16 | export ENABLE_HTTPS=1 17 | export SERVER_REGION=us-east-1 18 | ./run.sh /tmp/output.log /tmp/error.log 19 | ``` 20 | -------------------------------------------------------------------------------- /run/core/s3cmd/README.md: -------------------------------------------------------------------------------- 1 | ## `s3cmd` tests 2 | This directory serves as the location for Mint tests using `s3cmd`. Top level `mint.sh` calls `run.sh` to execute tests. 3 | 4 | ## Adding new tests 5 | New tests is added into `test.sh` as new functions. 6 | 7 | ## Running tests manually 8 | - Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` 9 | - Call `run.sh` with output log file and error log file. for example 10 | ```bash 11 | export MINT_DATA_DIR=~/my-mint-dir 12 | export MINT_MODE=core 13 | export SERVER_ENDPOINT="play.minio.io:9000" 14 | export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" 15 | export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" 16 | export ENABLE_HTTPS=1 17 | export SERVER_REGION=us-east-1 18 | ./run.sh /tmp/output.log /tmp/error.log 19 | ``` 20 | -------------------------------------------------------------------------------- /run/core/awscli/README.md: -------------------------------------------------------------------------------- 1 | ## `awscli` tests 2 | This directory serves as the location for Mint tests using `awscli`. Top level `mint.sh` calls `run.sh` to execute tests. 3 | 4 | ## Adding new tests 5 | New tests is added into `test.sh` as new functions. 6 | 7 | ## Running tests manually 8 | - Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` 9 | - Call `run.sh` with output log file and error log file. for example 10 | ```bash 11 | export MINT_DATA_DIR=~/my-mint-dir 12 | export MINT_MODE=core 13 | export SERVER_ENDPOINT="play.minio.io:9000" 14 | export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" 15 | export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" 16 | export ENABLE_HTTPS=1 17 | export SERVER_REGION=us-east-1 18 | ./run.sh /tmp/output.log /tmp/error.log 19 | ``` 20 | -------------------------------------------------------------------------------- /run/core/aws-sdk-php/README.md: -------------------------------------------------------------------------------- 1 | ## `aws-sdk-php` tests 2 | This directory serves as the location for Mint tests using `aws-sdk-php`. Top level `mint.sh` calls `run.sh` to execute tests. 3 | 4 | ## Adding new tests 5 | New tests is added into `quick-tests.php` as new functions. 6 | 7 | ## Running tests manually 8 | - Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` 9 | - Call `run.sh` with output log file and error log file. for example 10 | ```bash 11 | export MINT_DATA_DIR=~/my-mint-dir 12 | export MINT_MODE=core 13 | export SERVER_ENDPOINT="play.minio.io:9000" 14 | export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" 15 | export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" 16 | export ENABLE_HTTPS=1 17 | export SERVER_REGION=us-east-1 18 | ./run.sh /tmp/output.log /tmp/error.log 19 | ``` 20 | -------------------------------------------------------------------------------- /run/core/aws-sdk-ruby/README.md: -------------------------------------------------------------------------------- 1 | ## `aws-sdk-ruby` tests 2 | This directory serves as the location for Mint tests using `aws-sdk-ruby`. Top level `mint.sh` calls `run.sh` to execute tests. 3 | 4 | ## Adding new tests 5 | New tests is added into `aws-stub-test.rb` as new functions. 6 | 7 | ## Running tests manually 8 | - Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` 9 | - Call `run.sh` with output log file and error log file. for example 10 | ```bash 11 | export MINT_DATA_DIR=~/my-mint-dir 12 | export MINT_MODE=core 13 | export SERVER_ENDPOINT="play.minio.io:9000" 14 | export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" 15 | export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" 16 | export ENABLE_HTTPS=1 17 | export SERVER_REGION=us-east-1 18 | ./run.sh /tmp/output.log /tmp/error.log 19 | ``` 20 | -------------------------------------------------------------------------------- /run/core/s3select/README.md: -------------------------------------------------------------------------------- 1 | 2 | ## `s3select` tests 3 | This directory serves as the location for Mint tests for s3select features. Top level `mint.sh` calls `run.sh` to execute tests. 4 | 5 | ## Adding new tests 6 | New tests are added into `s3select/tests.py` as new functions. 7 | 8 | ## Running tests manually 9 | - Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` 10 | - Call `run.sh` with output log file and error log file. for example 11 | 12 | ```bash 13 | export MINT_DATA_DIR=~/my-mint-dir 14 | export MINT_MODE=core 15 | export SERVER_ENDPOINT="play.min.io" 16 | export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" 17 | export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" 18 | export ENABLE_HTTPS=1 19 | export SERVER_REGION=us-east-1 20 | ./run.sh /tmp/output.log /tmp/error.log 21 | ``` 22 | -------------------------------------------------------------------------------- /run/core/minio-js/README.md: -------------------------------------------------------------------------------- 1 | ## `minio-js` tests 2 | This directory serves as the location for Mint tests using `minio-js`. Top level `mint.sh` calls `run.sh` to execute tests. 3 | 4 | ## Adding new tests 5 | New tests is added in functional tests of minio-js. Please check https://github.com/minio/minio-js 6 | 7 | ## Running tests manually 8 | - Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` 9 | - Call `run.sh` with output log file and error log file. for example 10 | ```bash 11 | export MINT_DATA_DIR=~/my-mint-dir 12 | export MINT_MODE=core 13 | export SERVER_ENDPOINT="play.minio.io:9000" 14 | export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" 15 | export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" 16 | export ENABLE_HTTPS=1 17 | export SERVER_REGION=us-east-1 18 | ./run.sh /tmp/output.log /tmp/error.log 19 | ``` 20 | -------------------------------------------------------------------------------- /run/core/minio-py/README.md: -------------------------------------------------------------------------------- 1 | ## `minio-py` tests 2 | This directory serves as the location for Mint tests using `minio-py`. Top level `mint.sh` calls `run.sh` to execute tests. 3 | 4 | ## Adding new tests 5 | New tests is added in functional tests of minio-py. Please check https://github.com/minio/minio-py 6 | 7 | ## Running tests manually 8 | - Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS` 9 | - Call `run.sh` with output log file and error log file. for example 10 | ```bash 11 | export MINT_DATA_DIR=~/my-mint-dir 12 | export MINT_MODE=core 13 | export SERVER_ENDPOINT="play.minio.io:9000" 14 | export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" 15 | export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" 16 | export ENABLE_HTTPS=1 17 | export SERVER_REGION=us-east-1 18 | ./run.sh /tmp/output.log /tmp/error.log 19 | ``` 20 | -------------------------------------------------------------------------------- /entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | ./mint.sh "$@" & 19 | 20 | # Get the pid to be used for kill command if required 21 | main_pid="$!" 22 | trap 'echo -e "\nAborting Mint..."; kill $main_pid' SIGINT SIGTERM 23 | # use -n here to catch mint.sh exit code, notify to ci 24 | wait -n 25 | -------------------------------------------------------------------------------- /postinstall.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | export APT="apt --quiet --yes" 19 | 20 | # remove all packages listed in remove-packages.list 21 | xargs --arg-file="${MINT_ROOT_DIR}/remove-packages.list" apt --quiet --yes purge 22 | ${APT} autoremove 23 | 24 | # flush to disk 25 | sync 26 | -------------------------------------------------------------------------------- /run/core/minio-go/README.md: -------------------------------------------------------------------------------- 1 | ## `minio-go` tests 2 | This directory serves as the location for Mint tests using `minio-go`. Top level `mint.sh` calls `run.sh` to execute tests. 3 | 4 | ## Adding new tests 5 | New tests are added in functional tests of minio-go. Please check https://github.com/minio/minio-go 6 | 7 | ## Running tests manually 8 | - Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION`, `ENABLE_HTTPS` and `RUN_ON_FAIL` 9 | - Call `run.sh` with output log file and error log file. for example 10 | ```bash 11 | export MINT_DATA_DIR=~/my-mint-dir 12 | export MINT_MODE=core 13 | export SERVER_ENDPOINT="play.minio.io:9000" 14 | export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" 15 | export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" 16 | export ENABLE_HTTPS=1 17 | export SERVER_REGION=us-east-1 18 | export RUN_ON_FAIL=1 19 | ./run.sh /tmp/output.log /tmp/error.log 20 | ``` 21 | -------------------------------------------------------------------------------- /run/core/minio-js/babel-register.js: -------------------------------------------------------------------------------- 1 | // fix babel register doesn't transform TypeScript 2 | // 3 | // https://github.com/babel/babel/issues/8962#issuecomment-443135379 4 | 5 | // eslint-disable-next-line @typescript-eslint/no-var-requires,import/no-commonjs 6 | const register = require('@babel/register') 7 | 8 | register({ 9 | extensions: ['.ts', '.js'], 10 | assumptions: { 11 | constantSuper: true, 12 | noIncompleteNsImportDetection: true, 13 | constantReexports: true, 14 | }, 15 | plugins: [ 16 | [ 17 | '@babel/plugin-transform-modules-commonjs', 18 | { 19 | importInterop: 'node', 20 | }, 21 | ], 22 | '@upleveled/remove-node-prefix', // lower version of node (<14) doesn't support require('node:fs') 23 | ], 24 | presets: [ 25 | ['@babel/preset-typescript', { allExtensions: true }], 26 | ['@babel/preset-env', { targets: { node: 'current' }, modules: 'cjs' }], 27 | ], 28 | }) 29 | -------------------------------------------------------------------------------- /run/core/minio-java/README.md: -------------------------------------------------------------------------------- 1 | ## `minio-java` tests 2 | This directory serves as the location for Mint tests using `minio-java`. Top level `mint.sh` calls `run.sh` to execute tests. 3 | 4 | ## Adding new tests 5 | New tests is added in functional tests of minio-java. Please check https://github.com/minio/minio-java 6 | 7 | ## Running tests manually 8 | - Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION`, `ENABLE_HTTPS` and `RUN_ON_FAIL` 9 | - Call `run.sh` with output log file and error log file. for example 10 | ```bash 11 | export MINT_DATA_DIR=~/my-mint-dir 12 | export MINT_MODE=core 13 | export SERVER_ENDPOINT="play.minio.io:9000" 14 | export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" 15 | export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" 16 | export ENABLE_HTTPS=1 17 | export SERVER_REGION=us-east-1 18 | export RUN_ON_FAIL=1 19 | ./run.sh /tmp/output.log /tmp/error.log 20 | ``` 21 | -------------------------------------------------------------------------------- /run/core/s3cmd/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # run tests 28 | ./test.sh 1>>"$output_log_file" 2>"$error_log_file" 29 | -------------------------------------------------------------------------------- /release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Minio Cloud Storage, (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | export MINT_ROOT_DIR=${MINT_ROOT_DIR:-/mint} 19 | source "${MINT_ROOT_DIR}"/source.sh 20 | 21 | # install mint app packages 22 | for pkg in "$MINT_ROOT_DIR/build"/*/install.sh; do 23 | echo "Running $pkg" 24 | $pkg 25 | done 26 | 27 | "${MINT_ROOT_DIR}"/postinstall.sh 28 | -------------------------------------------------------------------------------- /run/core/mc/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Minio Cloud Storage, (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | ./functional-tests.sh 1>>"$output_log_file" 2>"$error_log_file" 28 | -------------------------------------------------------------------------------- /run/core/aws-sdk-php/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint, (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # run tests 28 | php ./quick-tests.php 1>>"$output_log_file" 2>"$error_log_file" 29 | -------------------------------------------------------------------------------- /run/core/minio-go/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # run tests 28 | /mint/run/core/minio-go/minio-go 1>>"$output_log_file" 2>"$error_log_file" 29 | -------------------------------------------------------------------------------- /run/core/s3select/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2020 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # run path style tests 28 | python "./tests.py" 1>>"$output_log_file" 2>"$error_log_file" 29 | -------------------------------------------------------------------------------- /run/core/.minio-dotnet/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | /mint/run/core/minio-dotnet/out/Minio.Functional.Tests 1>>"$output_log_file" 2>"$error_log_file" 27 | -------------------------------------------------------------------------------- /run/core/versioning/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2021 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # run tests 28 | /mint/run/core/versioning/tests 1>>"$output_log_file" 2>"$error_log_file" 29 | -------------------------------------------------------------------------------- /.github/workflows/dependency-review.yml: -------------------------------------------------------------------------------- 1 | # Dependency Review Action 2 | # 3 | # This Action will scan dependency manifest files that change as part of a Pull Request, surfacing known-vulnerable versions of the packages declared or updated in the PR. Once installed, if the workflow run is marked as required, PRs introducing known-vulnerable packages will be blocked from merging. 4 | # 5 | # Source repository: https://github.com/actions/dependency-review-action 6 | # Public documentation: https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/about-dependency-review#dependency-review-enforcement 7 | name: 'Dependency Review' 8 | on: [pull_request] 9 | 10 | permissions: 11 | contents: read 12 | 13 | jobs: 14 | dependency-review: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: 'Checkout Repository' 18 | uses: actions/checkout@v3 19 | - name: 'Dependency Review' 20 | uses: actions/dependency-review-action@v1 21 | -------------------------------------------------------------------------------- /run/core/healthcheck/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2019 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # run tests 28 | /mint/run/core/healthcheck/healthcheck 1>>"$output_log_file" 2>"$error_log_file" 29 | -------------------------------------------------------------------------------- /run/core/minio-py/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # run tests 28 | python "/mint/run/core/minio-py/tests.py" 1>>"$output_log_file" 2>"$error_log_file" 29 | -------------------------------------------------------------------------------- /run/core/aws-sdk-go-v2/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2017-2025 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # run tests 28 | /mint/run/core/aws-sdk-go-v2/aws-sdk-go-v2 1>>"$output_log_file" 2>"$error_log_file" 29 | -------------------------------------------------------------------------------- /run/core/aws-sdk-ruby/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # run tests 28 | chmod a+x aws-stub-tests.rb 29 | ruby aws-stub-tests.rb 1>>"$output_log_file" 2>"$error_log_file" 30 | -------------------------------------------------------------------------------- /run/core/aws-sdk-java-v2/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2018 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # run tests 28 | cd /mint/run/core/aws-sdk-java-v2/ || exit 1 29 | 30 | java -jar FunctionalTests.jar 1>>"$output_log_file" 2>"$error_log_file" 31 | -------------------------------------------------------------------------------- /run/core/minio-js/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Minio Cloud Storage, (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # Prepare test runner project 28 | cp -R /mint/test-run/minio-js/ ./minio-js 29 | npm install --quiet &>/dev/null 30 | 31 | # Run tests 32 | node ./node_modules/mocha/bin/mocha.js "./minio-js/tests/functional/functional-tests.js" -R minioreporter >>"$output_log_file" 2>"$error_log_file" 33 | -------------------------------------------------------------------------------- /run/core/minio-java/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # run tests 28 | endpoint="http://$SERVER_ENDPOINT" 29 | if [ "$ENABLE_HTTPS" -eq 1 ]; then 30 | endpoint="https://$SERVER_ENDPOINT" 31 | fi 32 | 33 | java -Xmx4096m -Xms256m -cp "/mint/run/core/minio-java/*:." FunctionalTest \ 34 | "$endpoint" "$ACCESS_KEY" "$SECRET_KEY" "$SERVER_REGION" 1>>"$output_log_file" 2>"$error_log_file" 35 | -------------------------------------------------------------------------------- /source.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Minio Cloud Storage, (C) 2024 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | export MINT_RUN_CORE_DIR="$MINT_ROOT_DIR/run/core" 19 | export MINT_RUN_BUILD_DIR="$MINT_ROOT_DIR/build" 20 | export APT="apt --quiet --yes" 21 | export WGET="wget --quiet --no-check-certificate" 22 | export WGET="wget --quiet --no-check-certificate" 23 | 24 | ## Software versions 25 | export GO_VERSION="1.25.4" 26 | export GRADLE_VERSION="9.2.0" 27 | export GRADLE_INSTALL_PATH="/opt/gradle" 28 | export GO_INSTALL_PATH="/usr/local" 29 | 30 | export PATH=${GO_INSTALL_PATH}/bin:$PATH 31 | export PATH=${GRADLE_INSTALL_PATH}/gradle-${GRADLE_VERSION}/bin:$PATH 32 | -------------------------------------------------------------------------------- /run/core/aws-sdk-go-v2/go.mod: -------------------------------------------------------------------------------- 1 | module mint.minio.io/aws-sdk-go-v2 2 | 3 | go 1.25.0 4 | 5 | require ( 6 | github.com/aws/aws-sdk-go-v2 v1.39.6 7 | github.com/aws/aws-sdk-go-v2/config v1.31.17 8 | github.com/aws/aws-sdk-go-v2/credentials v1.18.21 9 | github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0 10 | github.com/sirupsen/logrus v1.9.3 11 | ) 12 | 13 | require ( 14 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 // indirect 15 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13 // indirect 16 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 // indirect 17 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 // indirect 18 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect 19 | github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 // indirect 20 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 // indirect 21 | github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 // indirect 22 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 // indirect 23 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 // indirect 24 | github.com/aws/aws-sdk-go-v2/service/sso v1.30.1 // indirect 25 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.5 // indirect 26 | github.com/aws/aws-sdk-go-v2/service/sts v1.39.1 // indirect 27 | github.com/aws/smithy-go v1.23.2 // indirect 28 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect 29 | ) 30 | -------------------------------------------------------------------------------- /run/core/healthcheck/go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= 5 | github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= 6 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 7 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 8 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 9 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 10 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 11 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 12 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 13 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 14 | golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= 15 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 16 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 17 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 18 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 19 | -------------------------------------------------------------------------------- /create-data-files.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | MINT_DATA_DIR="$MINT_ROOT_DIR/data" 19 | 20 | declare -A data_file_map 21 | data_file_map["datafile-0-b"]="0" 22 | data_file_map["datafile-1-b"]="1" 23 | data_file_map["datafile-1-kB"]="1K" 24 | data_file_map["datafile-10-kB"]="10K" 25 | data_file_map["datafile-33-kB"]="33K" 26 | data_file_map["datafile-100-kB"]="100K" 27 | data_file_map["datafile-1.03-MB"]="1056K" 28 | data_file_map["datafile-1-MB"]="1M" 29 | data_file_map["datafile-5-MB"]="5M" 30 | data_file_map["datafile-5243880-b"]="5243880" 31 | data_file_map["datafile-6-MB"]="6M" 32 | data_file_map["datafile-10-MB"]="10M" 33 | data_file_map["datafile-11-MB"]="11M" 34 | data_file_map["datafile-65-MB"]="65M" 35 | data_file_map["datafile-129-MB"]="129M" 36 | 37 | mkdir -p "$MINT_DATA_DIR" 38 | for filename in "${!data_file_map[@]}"; do 39 | echo "creating $MINT_DATA_DIR/$filename" 40 | if ! shred -n 1 -s "${data_file_map[$filename]}" - 1>"$MINT_DATA_DIR/$filename" 2>/dev/null; then 41 | echo "unable to create data file $MINT_DATA_DIR/$filename" 42 | exit 1 43 | fi 44 | done 45 | -------------------------------------------------------------------------------- /run/core/awscli/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # handle command line arguments 19 | if [ $# -ne 2 ]; then 20 | echo "usage: run.sh " 21 | exit 1 22 | fi 23 | 24 | output_log_file="$1" 25 | error_log_file="$2" 26 | 27 | # Enable botocore's empty body handling for Expect: 100-continue 28 | export BOTO_EXPERIMENTAL__NO_EMPTY_CONTINUE=true 29 | 30 | # configure awscli 31 | aws configure set aws_access_key_id "$ACCESS_KEY" 32 | aws configure set aws_secret_access_key "$SECRET_KEY" 33 | aws configure set default.region "$SERVER_REGION" 34 | 35 | # run tests for virtual style if provided 36 | if [ "$ENABLE_VIRTUAL_STYLE" -eq 1 ]; then 37 | # Setup endpoint scheme 38 | endpoint="http://$DOMAIN:$SERVER_PORT" 39 | if [ "$ENABLE_HTTPS" -eq 1 ]; then 40 | endpoint="https://$DOMAIN:$SERVER_PORT" 41 | fi 42 | dnsmasq --address="/$DOMAIN/$SERVER_IP" --user=root 43 | echo -e "nameserver 127.0.0.1\n$(cat /etc/resolv.conf)" >/etc/resolv.conf 44 | aws configure set default.s3.addressing_style virtual 45 | ./test.sh "$endpoint" 1>>"$output_log_file" 2>"$error_log_file" 46 | aws configure set default.s3.addressing_style path 47 | fi 48 | 49 | endpoint="http://$SERVER_ENDPOINT" 50 | if [ "$ENABLE_HTTPS" -eq 1 ]; then 51 | endpoint="https://$SERVER_ENDPOINT" 52 | fi 53 | # run path style tests 54 | ./test.sh "$endpoint" 1>>"$output_log_file" 2>"$error_log_file" 55 | -------------------------------------------------------------------------------- /run/core/minio-js/minioreporter.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Reporter for JSON formatted logging, (C) 2017 Minio, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | var mocha = require('mocha'); 18 | module.exports = minioreporter; 19 | 20 | function minioreporter(runner) { 21 | mocha.reporters.Base.call(this, runner); 22 | var self = this; 23 | 24 | runner.on('pass', function (test) { 25 | GenerateJsonEntry(test) 26 | }); 27 | 28 | runner.on('fail', function (test, err) { 29 | GenerateJsonEntry(test, err) 30 | }); 31 | 32 | } 33 | 34 | /** 35 | * Convert test result into a JSON object and print on the console. 36 | * 37 | * @api private 38 | * @param test, err 39 | */ 40 | 41 | function GenerateJsonEntry (test, err) { 42 | var res = test.title.split("_") 43 | var jsonEntry = {}; 44 | 45 | jsonEntry.name = "minio-js" 46 | 47 | if (res.length > 0 && res[0].length) { 48 | jsonEntry.function = res[0] 49 | } 50 | 51 | if (res.length > 1 && res[1].length) { 52 | jsonEntry.args = res[1] 53 | } 54 | 55 | jsonEntry.duration = test.duration 56 | 57 | if (res.length > 2 && res[2].length) { 58 | jsonEntry.alert = res[2] 59 | } 60 | 61 | if (err != null ) { 62 | jsonEntry.status = "FAIL" 63 | jsonEntry.error = err.stack.replace(/\n/g, " ").replace(/ +(?= )/g,'') 64 | } else { 65 | jsonEntry.status = "PASS" 66 | } 67 | 68 | process.stdout.write(JSON.stringify(jsonEntry) + "\n") 69 | } 70 | -------------------------------------------------------------------------------- /preinstall.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Mint (C) 2017-2022 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | source "${MINT_ROOT_DIR}"/source.sh 19 | 20 | # install nodejs source list 21 | if ! $WGET --output-document=- https://deb.nodesource.com/setup_24.x | bash -; then 22 | echo "unable to set nodejs repository" 23 | exit 1 24 | fi 25 | 26 | $APT install apt-transport-https 27 | 28 | # Ubuntu 24.04 provides .NET SDK directly from Ubuntu repos 29 | # No need for Microsoft package repository anymore 30 | 31 | $APT update 32 | $APT install gnupg ca-certificates unzip busybox 33 | 34 | # download and install golang 35 | download_url="https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" 36 | if ! $WGET --output-document=- "$download_url" | tar -C "${GO_INSTALL_PATH}" -zxf -; then 37 | echo "unable to install go$GO_VERSION" 38 | exit 1 39 | fi 40 | 41 | xargs --arg-file="${MINT_ROOT_DIR}/install-packages.list" apt --quiet --yes install 42 | 43 | # set python 3.12 as default (Ubuntu 24.04) 44 | update-alternatives --install /usr/bin/python python /usr/bin/python3.12 1 45 | 46 | mkdir -p ${GRADLE_INSTALL_PATH} 47 | gradle_url="https://services.gradle.org/distributions/gradle-${GRADLE_VERSION}-bin.zip" 48 | if ! $WGET --output-document=- "$gradle_url" | busybox unzip -qq -d ${GRADLE_INSTALL_PATH} -; then 49 | echo "unable to install gradle-${GRADLE_VERSION}" 50 | exit 1 51 | fi 52 | 53 | chmod +x -v ${GRADLE_INSTALL_PATH}/gradle-${GRADLE_VERSION}/bin/gradle 54 | 55 | sync 56 | -------------------------------------------------------------------------------- /run/core/minio-js/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bin", 3 | "version": "1.0.0", 4 | "keywords": ["mint minio js"], 5 | "author": "", 6 | "license": "ISC", 7 | "description": "", 8 | "dependencies": { 9 | "async": "^3.2.6", 10 | "block-stream2": "^2.1.0", 11 | "browser-or-node": "^2.1.1", 12 | "buffer-crc32": "^0.2.13", 13 | "fast-xml-parser": "^4.5.0", 14 | "ipaddr.js": "^2.0.1", 15 | "json-stream": "^1.0.0", 16 | "lodash": "^4.17.21", 17 | "mime-types": "^2.1.35", 18 | "query-string": "^7.1.3", 19 | "through2": "^4.0.2", 20 | "web-encoding": "^1.1.5", 21 | "xml": "^1.0.1", 22 | "xml2js": "^0.6.2" 23 | }, 24 | "devDependencies": { 25 | "@babel/core": "^7.21.8", 26 | "@babel/plugin-transform-modules-commonjs": "^7.21.5", 27 | "@babel/preset-env": "^7.21.5", 28 | "@babel/preset-typescript": "^7.21.5", 29 | "@babel/register": "^7.21.0", 30 | "@nodelib/fs.walk": "^1.2.8", 31 | "@types/async": "^3.2.20", 32 | "@types/lodash": "^4.14.194", 33 | "@types/mime-types": "^2.1.1", 34 | "@types/node": "^20.1.0", 35 | "@types/xml": "^1.0.8", 36 | "@types/xml2js": "^0.4.11", 37 | "@typescript-eslint/eslint-plugin": "^5.59.2", 38 | "@typescript-eslint/parser": "^5.59.2", 39 | "@upleveled/babel-plugin-remove-node-prefix": "^1.0.5", 40 | "babel-plugin-replace-import-extension": "^1.1.3", 41 | "babel-plugin-transform-replace-expressions": "^0.2.0", 42 | "chai": "^4.3.7", 43 | "dotenv": "^16.0.3", 44 | "eslint": "^8.40.0", 45 | "eslint-config-prettier": "^8.8.0", 46 | "eslint-import-resolver-typescript": "^3.5.5", 47 | "eslint-plugin-import": "^2.27.5", 48 | "eslint-plugin-simple-import-sort": "^10.0.0", 49 | "eslint-plugin-unicorn": "^47.0.0", 50 | "eslint-plugin-unused-imports": "^2.0.0", 51 | "husky": "^8.0.3", 52 | "lint-staged": "^13.2.2", 53 | "mocha": "^10.2.0", 54 | "mocha-steps": "^1.3.0", 55 | "nock": "^13.3.1", 56 | "prettier": "^2.8.8", 57 | "source-map-support": "^0.5.21", 58 | "split-file": "^2.3.0", 59 | "superagent": "^8.0.1", 60 | "typescript": "^5.0.4", 61 | "uuid": "^9.0.0" 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ "master" ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ "master" ] 20 | schedule: 21 | - cron: '25 7 * * 0' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'go', 'javascript', 'python', 'ruby' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v3 42 | 43 | # Initializes the CodeQL tools for scanning. 44 | - name: Initialize CodeQL 45 | uses: github/codeql-action/init@v2 46 | with: 47 | languages: ${{ matrix.language }} 48 | # If you wish to specify custom queries, you can do so here or in a config file. 49 | # By default, queries listed here will override any specified in a config file. 50 | # Prefix the list here with "+" to use these queries and those in the config file. 51 | 52 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 53 | # queries: security-extended,security-and-quality 54 | 55 | 56 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 57 | # If this step fails, then you should remove it and run the build manually (see below) 58 | - name: Autobuild 59 | uses: github/codeql-action/autobuild@v2 60 | 61 | # ℹ️ Command-line programs to run using the OS shell. 62 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 63 | 64 | # If the Autobuild fails above, remove it and uncomment the following three lines. 65 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 66 | 67 | # - run: | 68 | # echo "Run, Build Application using script" 69 | # ./location_of_script_within_repo/buildscript.sh 70 | 71 | - name: Perform CodeQL Analysis 72 | uses: github/codeql-action/analyze@v2 73 | -------------------------------------------------------------------------------- /run/core/s3select/tests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # MinIO Python Library for Amazon S3 Compatible Cloud Storage, 4 | # (C) 2015-2020 MinIO, Inc. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | import os 19 | import sys 20 | from csv import (test_csv_input_custom_quote_char, 21 | test_csv_output_custom_quote_char) 22 | 23 | from minio import Minio 24 | 25 | from sql_ops import (test_sql_datatypes, test_sql_functions_agg_cond_conv, 26 | test_sql_functions_date, test_sql_functions_string, 27 | test_sql_operators, test_sql_operators_precedence, 28 | test_sql_select, test_sql_select_csv_no_header, 29 | test_sql_select_json) 30 | from utils import LogOutput 31 | 32 | 33 | def main(): 34 | """ 35 | Functional testing for S3 select. 36 | """ 37 | 38 | try: 39 | access_key = os.getenv('ACCESS_KEY', 'Q3AM3UQ867SPQQA43P2F') 40 | secret_key = os.getenv('SECRET_KEY', 41 | 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG') 42 | server_endpoint = os.getenv('SERVER_ENDPOINT', 'play.min.io') 43 | secure = os.getenv('ENABLE_HTTPS', '1') == '1' 44 | if server_endpoint == 'play.min.io': 45 | access_key = 'Q3AM3UQ867SPQQA43P2F' 46 | secret_key = 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG' 47 | secure = True 48 | 49 | client = Minio( 50 | endpoint=server_endpoint, 51 | access_key=access_key, 52 | secret_key=secret_key, 53 | secure=secure 54 | ) 55 | 56 | log_output = LogOutput(client.select_object_content, 57 | 'test_csv_input_quote_char') 58 | test_csv_input_custom_quote_char(client, log_output) 59 | 60 | log_output = LogOutput(client.select_object_content, 61 | 'test_csv_output_quote_char') 62 | test_csv_output_custom_quote_char(client, log_output) 63 | 64 | log_output = LogOutput( 65 | client.select_object_content, 'test_sql_operators') 66 | test_sql_operators(client, log_output) 67 | 68 | log_output = LogOutput(client.select_object_content, 69 | 'test_sql_operators_precedence') 70 | test_sql_operators_precedence(client, log_output) 71 | 72 | log_output = LogOutput(client.select_object_content, 73 | 'test_sql_functions_agg_cond_conv') 74 | test_sql_functions_agg_cond_conv(client, log_output) 75 | 76 | log_output = LogOutput( 77 | client.select_object_content, 'test_sql_functions_date') 78 | test_sql_functions_date(client, log_output) 79 | 80 | log_output = LogOutput(client.select_object_content, 81 | 'test_sql_functions_string') 82 | test_sql_functions_string(client, log_output) 83 | 84 | log_output = LogOutput( 85 | client.select_object_content, 'test_sql_datatypes') 86 | test_sql_datatypes(client, log_output) 87 | 88 | log_output = LogOutput(client.select_object_content, 'test_sql_select') 89 | test_sql_select(client, log_output) 90 | 91 | log_output = LogOutput( 92 | client.select_object_content, 'test_sql_select_json') 93 | test_sql_select_json(client, log_output) 94 | 95 | log_output = LogOutput( 96 | client.select_object_content, 'test_sql_select_csv') 97 | test_sql_select_csv_no_header(client, log_output) 98 | 99 | except Exception as err: 100 | print(log_output.json_report(err)) 101 | sys.exit(1) 102 | 103 | 104 | if __name__ == "__main__": 105 | # Execute only if run as a script 106 | main() 107 | -------------------------------------------------------------------------------- /run/core/s3select/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # MinIO Python Library for Amazon S3 Compatible Cloud Storage, 4 | # (C) 2015-2020 MinIO, Inc. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | import inspect 19 | import json 20 | import time 21 | import traceback 22 | import uuid 23 | 24 | 25 | class LogOutput(object): 26 | """ 27 | LogOutput is the class for log output. It is required standard for all 28 | SDK tests controlled by mint. 29 | Here are its attributes: 30 | 'name': name of the SDK under test, e.g. 's3select' 31 | 'function': name of the method/api under test with its signature 32 | The following python code can be used to 33 | pull args information of a and to 34 | put together with the method name: 35 | .__name__+'('+', '.join(args_list)+')' 36 | e.g. 'remove_object(bucket_name, object_name)' 37 | 'args': method/api arguments with their values, in 38 | dictionary form: {'arg1': val1, 'arg2': val2, ...} 39 | 'duration': duration of the whole test in milliseconds, 40 | defaults to 0 41 | 'alert': any extra information user is needed to be alerted about, 42 | like whether this is a Blocker/Gateway/Server related 43 | issue, etc., defaults to None 44 | 'message': descriptive error message, defaults to None 45 | 'error': stack-trace/exception message(only in case of failure), 46 | actual low level exception/error thrown by the program, 47 | defaults to None 48 | 'status': exit status, possible values are 'PASS', 'FAIL', 'NA', 49 | defaults to 'PASS' 50 | """ 51 | 52 | PASS = 'PASS' 53 | FAIL = 'FAIL' 54 | NA = 'NA' 55 | 56 | def __init__(self, meth, test_name): 57 | self.__args_list = inspect.getfullargspec(meth).args[1:] 58 | self.__name = 's3select:'+test_name 59 | self.__function = meth.__name__+'('+', '.join(self.__args_list)+')' 60 | self.__args = {} 61 | self.__duration = 0 62 | self.__alert = '' 63 | self.__message = None 64 | self.__error = None 65 | self.__status = self.PASS 66 | self.__start_time = time.time() 67 | 68 | @property 69 | def name(self): return self.__name 70 | 71 | @property 72 | def function(self): return self.__function 73 | 74 | @property 75 | def args(self): return self.__args 76 | 77 | @name.setter 78 | def name(self, val): self.__name = val 79 | 80 | @function.setter 81 | def function(self, val): self.__function = val 82 | 83 | @args.setter 84 | def args(self, val): self.__args = val 85 | 86 | def json_report(self, err_msg='', alert='', status=''): 87 | self.__args = {k: v for k, v in self.__args.items() if v and v != ''} 88 | entry = {'name': self.__name, 89 | 'function': self.__function, 90 | 'args': self.__args, 91 | 'duration': int(round((time.time() - self.__start_time)*1000)), 92 | 'alert': str(alert), 93 | 'message': str(err_msg), 94 | 'error': traceback.format_exc() if err_msg and err_msg != '' else '', 95 | 'status': status if status and status != '' else 96 | self.FAIL if err_msg and err_msg != '' else self.PASS 97 | } 98 | return json.dumps({k: v for k, v in entry.items() if v and v != ''}) 99 | 100 | 101 | def generate_bucket_name(): 102 | return "s3select-test-" + str(uuid.uuid4()) 103 | 104 | 105 | def generate_object_name(): 106 | return str(uuid.uuid4()) 107 | -------------------------------------------------------------------------------- /run/core/aws-sdk-go-v2/go.sum: -------------------------------------------------------------------------------- 1 | github.com/aws/aws-sdk-go-v2 v1.39.6 h1:2JrPCVgWJm7bm83BDwY5z8ietmeJUbh3O2ACnn+Xsqk= 2 | github.com/aws/aws-sdk-go-v2 v1.39.6/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE= 3 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0= 4 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3/go.mod h1:xdCzcZEtnSTKVDOmUZs4l/j3pSV6rpo1WXl5ugNsL8Y= 5 | github.com/aws/aws-sdk-go-v2/config v1.31.17 h1:QFl8lL6RgakNK86vusim14P2k8BFSxjvUkcWLDjgz9Y= 6 | github.com/aws/aws-sdk-go-v2/config v1.31.17/go.mod h1:V8P7ILjp/Uef/aX8TjGk6OHZN6IKPM5YW6S78QnRD5c= 7 | github.com/aws/aws-sdk-go-v2/credentials v1.18.21 h1:56HGpsgnmD+2/KpG0ikvvR8+3v3COCwaF4r+oWwOeNA= 8 | github.com/aws/aws-sdk-go-v2/credentials v1.18.21/go.mod h1:3YELwedmQbw7cXNaII2Wywd+YY58AmLPwX4LzARgmmA= 9 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13 h1:T1brd5dR3/fzNFAQch/iBKeX07/ffu/cLu+q+RuzEWk= 10 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13/go.mod h1:Peg/GBAQ6JDt+RoBf4meB1wylmAipb7Kg2ZFakZTlwk= 11 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 h1:a+8/MLcWlIxo1lF9xaGt3J/u3yOZx+CdSveSNwjhD40= 12 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13/go.mod h1:oGnKwIYZ4XttyU2JWxFrwvhF6YKiK/9/wmE3v3Iu9K8= 13 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 h1:HBSI2kDkMdWz4ZM7FjwE7e/pWDEZ+nR95x8Ztet1ooY= 14 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13/go.mod h1:YE94ZoDArI7awZqJzBAZ3PDD2zSfuP7w6P2knOzIn8M= 15 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= 16 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= 17 | github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 h1:eg/WYAa12vqTphzIdWMzqYRVKKnCboVPRlvaybNCqPA= 18 | github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13/go.mod h1:/FDdxWhz1486obGrKKC1HONd7krpk38LBt+dutLcN9k= 19 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o= 20 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo= 21 | github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 h1:NvMjwvv8hpGUILarKw7Z4Q0w1H9anXKsesMxtw++MA4= 22 | github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4/go.mod h1:455WPHSwaGj2waRSpQp7TsnpOnBfw8iDfPfbwl7KPJE= 23 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 h1:kDqdFvMY4AtKoACfzIGD8A0+hbT41KTKF//gq7jITfM= 24 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13/go.mod h1:lmKuogqSU3HzQCwZ9ZtcqOc5XGMqtDK7OIc2+DxiUEg= 25 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 h1:zhBJXdhWIFZ1acfDYIhu4+LCzdUS2Vbcum7D01dXlHQ= 26 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13/go.mod h1:JaaOeCE368qn2Hzi3sEzY6FgAZVCIYcC2nwbro2QCh8= 27 | github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0 h1:ef6gIJR+xv/JQWwpa5FYirzoQctfSJm7tuDe3SZsUf8= 28 | github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0/go.mod h1:+wArOOrcHUevqdto9k1tKOF5++YTe9JEcPSc9Tx2ZSw= 29 | github.com/aws/aws-sdk-go-v2/service/sso v1.30.1 h1:0JPwLz1J+5lEOfy/g0SURC9cxhbQ1lIMHMa+AHZSzz0= 30 | github.com/aws/aws-sdk-go-v2/service/sso v1.30.1/go.mod h1:fKvyjJcz63iL/ftA6RaM8sRCtN4r4zl4tjL3qw5ec7k= 31 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.5 h1:OWs0/j2UYR5LOGi88sD5/lhN6TDLG6SfA7CqsQO9zF0= 32 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.5/go.mod h1:klO+ejMvYsB4QATfEOIXk8WAEwN4N0aBfJpvC+5SZBo= 33 | github.com/aws/aws-sdk-go-v2/service/sts v1.39.1 h1:mLlUgHn02ue8whiR4BmxxGJLR2gwU6s6ZzJ5wDamBUs= 34 | github.com/aws/aws-sdk-go-v2/service/sts v1.39.1/go.mod h1:E19xDjpzPZC7LS2knI9E6BaRFDK43Eul7vd6rSq2HWk= 35 | github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM= 36 | github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= 37 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 38 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 39 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 40 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 41 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 42 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 43 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 44 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 45 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 46 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 47 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= 48 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 49 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 50 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 51 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 52 | -------------------------------------------------------------------------------- /mint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2017-2022 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | CONTAINER_ID=$(grep -o -e '[0-f]\{12,\}' /proc/1/cpuset | awk '{print substr($1, 1, 12)}') 19 | MINT_DATA_DIR=${MINT_DATA_DIR:-/mint/data} 20 | MINT_MODE=${MINT_MODE:-core} 21 | SERVER_REGION=${SERVER_REGION:-us-east-1} 22 | ENABLE_HTTPS=${ENABLE_HTTPS:-0} 23 | ENABLE_VIRTUAL_STYLE=${ENABLE_VIRTUAL_STYLE:-0} 24 | RUN_ON_FAIL=${RUN_ON_FAIL:-0} 25 | 26 | if [ -z "$SERVER_ENDPOINT" ]; then 27 | SERVER_ENDPOINT="play.minio.io:9000" 28 | ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" 29 | SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" 30 | ENABLE_HTTPS=1 31 | fi 32 | 33 | if [ "$ENABLE_VIRTUAL_STYLE" -eq 1 ]; then 34 | SERVER_IP="${SERVER_ENDPOINT%%:*}" 35 | SERVER_PORT="${SERVER_ENDPOINT/*:/}" 36 | # Check if SERVER_IP is actually IPv4 address 37 | IFS=. read -ra octets <<<"$SERVER_IP" 38 | if [ "${#octets[@]}" -ne 4 ]; then 39 | echo "$SERVER_IP must be an IP address" 40 | exit 1 41 | fi 42 | for octet in "${octets[@]}"; do 43 | if [ "$octet" -lt 0 ] 2>/dev/null || [ "$octet" -gt 255 ] 2>/dev/null; then 44 | echo "$SERVER_IP must be an IP address" 45 | exit 1 46 | fi 47 | done 48 | fi 49 | 50 | ROOT_DIR="$PWD" 51 | TESTS_DIR="$ROOT_DIR/run/core" 52 | 53 | BASE_LOG_DIR="$ROOT_DIR/log" 54 | LOG_FILE="log.json" 55 | ERROR_FILE="error.log" 56 | mkdir -p "$BASE_LOG_DIR" 57 | 58 | function humanize_time() { 59 | time="$1" 60 | days=$((time / 60 / 60 / 24)) 61 | hours=$((time / 60 / 60 % 24)) 62 | minutes=$((time / 60 % 60)) 63 | seconds=$((time % 60)) 64 | 65 | ((days > 0)) && echo -n "$days days " 66 | ((hours > 0)) && echo -n "$hours hours " 67 | ((minutes > 0)) && echo -n "$minutes minutes " 68 | ((days > 0 || hours > 0 || minutes > 0)) && echo -n "and " 69 | echo "$seconds seconds" 70 | } 71 | 72 | function run_test() { 73 | if [ ! -d "$1" ]; then 74 | return 1 75 | fi 76 | 77 | start=$(date +%s) 78 | 79 | mkdir -p "$BASE_LOG_DIR/$sdk_name" 80 | 81 | # Use per-test log file instead of shared log 82 | test_log_file="$BASE_LOG_DIR/$sdk_name/$LOG_FILE" 83 | 84 | (cd "$sdk_dir" && ./run.sh "$test_log_file" "$BASE_LOG_DIR/$sdk_name/$ERROR_FILE") 85 | rv=$? 86 | 87 | # Append test results to global log file 88 | if [ -f "$test_log_file" ]; then 89 | cat "$test_log_file" >>"$BASE_LOG_DIR/$LOG_FILE" 90 | fi 91 | 92 | end=$(date +%s) 93 | duration=$(humanize_time $((end - start))) 94 | 95 | if [ "$rv" -eq 0 ]; then 96 | echo "done in $duration" 97 | else 98 | echo "FAILED in $duration" 99 | # Read from test-specific log file, not shared one 100 | if [ -f "$test_log_file" ]; then 101 | entry=$(tail -n 1 "$test_log_file") 102 | else 103 | entry="" 104 | fi 105 | status=$(jq -e -r .status <<<"$entry" 2>/dev/null) 106 | jq_rv=$? 107 | if [ "$jq_rv" -ne 0 ]; then 108 | echo "$entry" 109 | fi 110 | ## Show error.log when status is empty or not "FAIL". 111 | ## This may happen when test run failed without providing logs. 112 | if [ "$jq_rv" -ne 0 ] || [ -z "$status" ] || { [ "$status" != "FAIL" ] && [ "$status" != "fail" ]; }; then 113 | cat "$BASE_LOG_DIR/$sdk_name/$ERROR_FILE" 114 | else 115 | jq . <<<"$entry" 116 | fi 117 | fi 118 | return $rv 119 | } 120 | 121 | function trust_s3_endpoint_tls_cert() { 122 | # Download the public certificate from the server 123 | openssl s_client -showcerts -verify 5 -connect "$SERVER_ENDPOINT" out}' 125 | for cert in *.pem; do 126 | cat "${cert}" >>/etc/ssl/certs/ca-certificates.crt 127 | done 128 | 129 | # Ask different SDKs/tools to load system certificates 130 | export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt 131 | export NODE_EXTRA_CA_CERTS=/etc/ssl/certs/ca-certificates.crt 132 | export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt 133 | export AWS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt 134 | } 135 | 136 | function main() { 137 | export MINT_DATA_DIR 138 | export MINT_MODE 139 | export SERVER_ENDPOINT 140 | export SERVER_IP 141 | export SERVER_PORT 142 | 143 | export ACCESS_KEY 144 | export SECRET_KEY 145 | export ENABLE_HTTPS 146 | export SERVER_REGION 147 | export ENABLE_VIRTUAL_STYLE 148 | export RUN_ON_FAIL 149 | 150 | echo "Running with" 151 | echo "SERVER_ENDPOINT: $SERVER_ENDPOINT" 152 | echo "ACCESS_KEY: $ACCESS_KEY" 153 | echo "SECRET_KEY: ***REDACTED***" 154 | echo "ENABLE_HTTPS: $ENABLE_HTTPS" 155 | echo "SERVER_REGION: $SERVER_REGION" 156 | echo "MINT_DATA_DIR: $MINT_DATA_DIR" 157 | echo "MINT_MODE: $MINT_MODE" 158 | echo "ENABLE_VIRTUAL_STYLE: $ENABLE_VIRTUAL_STYLE" 159 | echo "RUN_ON_FAIL: $RUN_ON_FAIL" 160 | echo 161 | echo "To get logs, run 'docker cp ${CONTAINER_ID}:/mint/log /tmp/mint-logs'" 162 | echo 163 | 164 | [ "$ENABLE_HTTPS" == "1" ] && trust_s3_endpoint_tls_cert 165 | 166 | declare -a run_list 167 | sdks=("$@") 168 | 169 | if [ "$#" -eq 0 ]; then 170 | cd "$TESTS_DIR" || exit 171 | sdks=(*) 172 | cd .. || exit 173 | fi 174 | 175 | for sdk in "${sdks[@]}"; do 176 | sdk=$(basename "$sdk") 177 | run_list=("${run_list[@]}" "$TESTS_DIR/$sdk") 178 | done 179 | 180 | count="${#run_list[@]}" 181 | i=0 182 | j=0 183 | for sdk_dir in "${run_list[@]}"; do 184 | sdk_name=$(basename "$sdk_dir") 185 | ((i++)) 186 | ((j++)) 187 | if [ ! -d "$sdk_dir" ]; then 188 | echo "Test $sdk_name not found. Exiting Mint." 189 | exit 1 190 | fi 191 | echo -n "($j/$count) Running $sdk_name tests ... " 192 | if ! run_test "$sdk_dir"; then 193 | ((i--)) 194 | fi 195 | done 196 | 197 | ## Report when all tests in run_list are run 198 | if [ "$i" -eq "$count" ]; then 199 | echo -e "\nAll tests ran successfully" 200 | else 201 | echo -e "\nExecuted $i out of $count tests successfully." 202 | exit 1 203 | fi 204 | } 205 | main "$@" 206 | -------------------------------------------------------------------------------- /run/core/minio-go/versioning_test.go: -------------------------------------------------------------------------------- 1 | // +build mint 2 | 3 | /* 4 | * 5 | * Mint, (C) 2025 Minio, Inc. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); 8 | * you may not use this file except in compliance with the License. 9 | * You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, 15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | * See the License for the specific language governing permissions and 17 | * limitations under the License. 18 | * 19 | */ 20 | 21 | package main 22 | 23 | import ( 24 | "context" 25 | "fmt" 26 | "math/rand" 27 | "os" 28 | "time" 29 | 30 | "github.com/minio/minio-go/v7" 31 | "github.com/minio/minio-go/v7/pkg/credentials" 32 | ) 33 | 34 | // testBucketVersioningExcludedPrefixes tests that excluded_prefixes are properly 35 | // set and retrieved via bucket versioning configuration APIs. 36 | // This test replicates the minio-py test_set_get_bucket_versioning test to verify 37 | // that EOS properly returns excluded_prefixes in GetBucketVersioning API response. 38 | func testBucketVersioningExcludedPrefixes() { 39 | startTime := time.Now() 40 | testName := "testBucketVersioningExcludedPrefixes" 41 | function := "GetBucketVersioning/SetBucketVersioning" 42 | 43 | // Initialize minio client 44 | endpoint := os.Getenv("SERVER_ENDPOINT") 45 | accessKey := os.Getenv("ACCESS_KEY") 46 | secretKey := os.Getenv("SECRET_KEY") 47 | secure := os.Getenv("ENABLE_HTTPS") == "1" 48 | 49 | c, err := minio.New(endpoint, &minio.Options{ 50 | Creds: credentials.NewStaticV4(accessKey, secretKey, ""), 51 | Secure: secure, 52 | }) 53 | if err != nil { 54 | logError(testName, function, nil, startTime, "", "MinIO client creation failed", err) 55 | return 56 | } 57 | 58 | // Generate unique bucket name 59 | bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 60 | args := map[string]interface{}{ 61 | "bucketName": bucketName, 62 | } 63 | 64 | ctx := context.Background() 65 | 66 | // Create bucket 67 | err = c.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{}) 68 | if err != nil { 69 | logError(testName, function, args, startTime, "", "MakeBucket failed", err) 70 | return 71 | } 72 | defer c.RemoveBucket(ctx, bucketName) 73 | 74 | // Test 1: Set versioning with excluded_prefixes 75 | excludedPrefixes := []minio.ExcludedPrefix{ 76 | {Prefix: "prefix1"}, 77 | {Prefix: "prefix2"}, 78 | } 79 | 80 | versioningConfig := minio.BucketVersioningConfiguration{ 81 | Status: "Enabled", 82 | ExcludedPrefixes: excludedPrefixes, 83 | ExcludeFolders: true, 84 | } 85 | 86 | err = c.SetBucketVersioning(ctx, bucketName, versioningConfig) 87 | if err != nil { 88 | logError(testName, function, args, startTime, "", fmt.Sprintf("SetBucketVersioning with excluded_prefixes failed: %v", err), err) 89 | return 90 | } 91 | 92 | // Get versioning configuration 93 | retrievedConfig, err := c.GetBucketVersioning(ctx, bucketName) 94 | if err != nil { 95 | logError(testName, function, args, startTime, "", "GetBucketVersioning failed", err) 96 | return 97 | } 98 | 99 | // Verify status 100 | if retrievedConfig.Status != "Enabled" { 101 | logError(testName, function, args, startTime, "", fmt.Sprintf("GetBucketVersioning status mismatch: expected 'Enabled', got '%s'", retrievedConfig.Status), nil) 102 | return 103 | } 104 | 105 | // Verify exclude_folders 106 | if !retrievedConfig.ExcludeFolders { 107 | logError(testName, function, args, startTime, "", fmt.Sprintf("GetBucketVersioning exclude_folders mismatch: expected true, got false"), nil) 108 | return 109 | } 110 | 111 | // Verify excluded_prefixes - THIS IS WHERE THE EOS BUG MANIFESTS 112 | if len(retrievedConfig.ExcludedPrefixes) != len(excludedPrefixes) { 113 | logError(testName, function, args, startTime, "", 114 | fmt.Sprintf("GetBucketVersioning excluded_prefixes count mismatch: expected %d, got %d. "+ 115 | "Expected: %v, Got: %v. "+ 116 | "EOS BUG: GetBucketVersioning returns empty excluded_prefixes array instead of configured values", 117 | len(excludedPrefixes), len(retrievedConfig.ExcludedPrefixes), 118 | excludedPrefixes, retrievedConfig.ExcludedPrefixes), nil) 119 | return 120 | } 121 | 122 | // Compare prefix values 123 | for i, expectedPrefix := range excludedPrefixes { 124 | if retrievedConfig.ExcludedPrefixes[i].Prefix != expectedPrefix.Prefix { 125 | logError(testName, function, args, startTime, "", 126 | fmt.Sprintf("GetBucketVersioning excluded_prefix[%d] mismatch: expected '%s', got '%s'", 127 | i, expectedPrefix.Prefix, retrievedConfig.ExcludedPrefixes[i].Prefix), nil) 128 | return 129 | } 130 | } 131 | 132 | // Test 2: Suspend versioning (should clear excluded_prefixes) 133 | suspendConfig := minio.BucketVersioningConfiguration{ 134 | Status: "Suspended", 135 | } 136 | 137 | err = c.SetBucketVersioning(ctx, bucketName, suspendConfig) 138 | if err != nil { 139 | logError(testName, function, args, startTime, "", "SetBucketVersioning suspend failed", err) 140 | return 141 | } 142 | 143 | // Get versioning configuration after suspend 144 | retrievedConfig2, err := c.GetBucketVersioning(ctx, bucketName) 145 | if err != nil { 146 | logError(testName, function, args, startTime, "", "GetBucketVersioning after suspend failed", err) 147 | return 148 | } 149 | 150 | // Verify status is suspended 151 | if retrievedConfig2.Status != "Suspended" { 152 | logError(testName, function, args, startTime, "", 153 | fmt.Sprintf("GetBucketVersioning status after suspend mismatch: expected 'Suspended', got '%s'", retrievedConfig2.Status), nil) 154 | return 155 | } 156 | 157 | // Verify exclude_folders is reset 158 | if retrievedConfig2.ExcludeFolders { 159 | logError(testName, function, args, startTime, "", 160 | fmt.Sprintf("GetBucketVersioning exclude_folders after suspend: expected false, got true"), nil) 161 | return 162 | } 163 | 164 | // Verify excluded_prefixes is empty 165 | if len(retrievedConfig2.ExcludedPrefixes) != 0 { 166 | logError(testName, function, args, startTime, "", 167 | fmt.Sprintf("GetBucketVersioning excluded_prefixes after suspend: expected empty, got %d prefixes: %v", 168 | len(retrievedConfig2.ExcludedPrefixes), retrievedConfig2.ExcludedPrefixes), nil) 169 | return 170 | } 171 | 172 | logSuccess(testName, function, args, startTime) 173 | } 174 | -------------------------------------------------------------------------------- /run/core/s3select/csv.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # MinIO Python Library for Amazon S3 Compatible Cloud Storage, 4 | # (C) 2015-2020 MinIO, Inc. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | import io 19 | import os 20 | 21 | from minio import Minio 22 | from minio.select import (COMPRESSION_TYPE_NONE, FILE_HEADER_INFO_NONE, 23 | JSON_TYPE_DOCUMENT, QUOTE_FIELDS_ALWAYS, 24 | QUOTE_FIELDS_ASNEEDED, CSVInputSerialization, 25 | CSVOutputSerialization, JSONInputSerialization, 26 | JSONOutputSerialization, SelectRequest) 27 | 28 | from utils import * 29 | 30 | 31 | def test_sql_api(test_name, client, bucket_name, input_data, sql_opts, expected_output): 32 | """ Test if the passed SQL request has the output equal to the passed execpted one""" 33 | object_name = generate_object_name() 34 | got_output = b'' 35 | try: 36 | bytes_content = io.BytesIO(input_data) 37 | client.put_object( 38 | bucket_name=bucket_name, 39 | object_name=object_name, 40 | data=io.BytesIO(input_data), 41 | length=len(input_data) 42 | ) 43 | data = client.select_object_content( 44 | bucket_name=bucket_name, 45 | object_name=object_name, 46 | request=sql_opts 47 | ) 48 | # Get the records 49 | records = io.BytesIO() 50 | for d in data.stream(10*1024): 51 | records.write(d) 52 | got_output = records.getvalue() 53 | except Exception as select_err: 54 | if not isinstance(expected_output, Exception): 55 | raise ValueError( 56 | 'Test {} unexpectedly failed with: {}'.format(test_name, select_err)) 57 | else: 58 | if isinstance(expected_output, Exception): 59 | raise ValueError( 60 | 'Test {}: expected an exception, got {}'.format(test_name, got_output)) 61 | if got_output != expected_output: 62 | raise ValueError('Test {}: data mismatch. Expected : {}, Received {}'.format( 63 | test_name, expected_output, got_output)) 64 | finally: 65 | client.remove_object(bucket_name=bucket_name, object_name=object_name) 66 | 67 | 68 | def test_csv_input_custom_quote_char(client, log_output): 69 | # Get a unique bucket_name and object_name 70 | log_output.args['bucket_name'] = bucket_name = generate_bucket_name() 71 | 72 | tests = [ 73 | # Invalid quote character, should fail 74 | ('""', '"', b'col1,col2,col3\n', Exception()), 75 | # UTF-8 quote character 76 | ('ع', '"', 'عcol1ع,عcol2ع,عcol3ع\n'.encode(), 77 | b'{"_1":"col1","_2":"col2","_3":"col3"}\n'), 78 | # Only one field is quoted 79 | ('"', '"', b'"col1",col2,col3\n', 80 | b'{"_1":"col1","_2":"col2","_3":"col3"}\n'), 81 | ('"', '"', b'"col1,col2,col3"\n', b'{"_1":"col1,col2,col3"}\n'), 82 | ('\'', '"', b'"col1",col2,col3\n', 83 | b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'), 84 | ('', '"', b'"col1",col2,col3\n', 85 | b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'), 86 | ('', '"', b'"col1",col2,col3\n', 87 | b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'), 88 | ('', '"', b'"col1","col2","col3"\n', 89 | b'{"_1":"\\"col1\\"","_2":"\\"col2\\"","_3":"\\"col3\\""}\n'), 90 | ('"', '"', b'""""""\n', b'{"_1":"\\"\\""}\n'), 91 | ('"', '"', b'A",B\n', b'{"_1":"A\\"","_2":"B"}\n'), 92 | ('"', '"', b'A"",B\n', b'{"_1":"A\\"\\"","_2":"B"}\n'), 93 | ('"', '\\', b'A\\B,C\n', b'{"_1":"A\\\\B","_2":"C"}\n'), 94 | ('"', '"', b'"A""B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'), 95 | ('"', '\\', b'"A\\B","CD"\n', b'{"_1":"AB","_2":"CD"}\n'), 96 | ('"', '\\', b'"A\\,","CD"\n', b'{"_1":"A,","_2":"CD"}\n'), 97 | ('"', '\\', b'"A\\"B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'), 98 | ('"', '\\', b'"A\\""\n', b'{"_1":"A\\""}\n'), 99 | ('"', '\\', b'"A\\"\\"B"\n', b'{"_1":"A\\"\\"B"}\n'), 100 | ('"', '\\', b'"A\\"","\\"B"\n', b'{"_1":"A\\"","_2":"\\"B"}\n'), 101 | ] 102 | 103 | client.make_bucket(bucket_name=bucket_name) 104 | 105 | try: 106 | for idx, (quote_char, escape_char, data, expected_output) in enumerate(tests): 107 | sql_opts = SelectRequest( 108 | "select * from s3object", 109 | CSVInputSerialization( 110 | compression_type=COMPRESSION_TYPE_NONE, 111 | file_header_info=FILE_HEADER_INFO_NONE, 112 | record_delimiter="\n", 113 | field_delimiter=",", 114 | quote_character=quote_char, 115 | quote_escape_character=escape_char, 116 | comments="#", 117 | allow_quoted_record_delimiter="FALSE", 118 | ), 119 | JSONOutputSerialization( 120 | record_delimiter="\n", 121 | ), 122 | request_progress=False, 123 | ) 124 | 125 | test_sql_api(f'test_{idx}', client, bucket_name, 126 | data, sql_opts, expected_output) 127 | finally: 128 | client.remove_bucket(bucket_name=bucket_name) 129 | 130 | # Test passes 131 | print(log_output.json_report()) 132 | 133 | 134 | def test_csv_output_custom_quote_char(client, log_output): 135 | # Get a unique bucket_name and object_name 136 | log_output.args['bucket_name'] = bucket_name = generate_bucket_name() 137 | 138 | tests = [ 139 | # UTF-8 quote character 140 | ("''", "''", b'col1,col2,col3\n', Exception()), 141 | ("'", "'", b'col1,col2,col3\n', b"'col1','col2','col3'\n"), 142 | ("", '"', b'col1,col2,col3\n', b'\x00col1\x00,\x00col2\x00,\x00col3\x00\n'), 143 | ('"', '"', b'col1,col2,col3\n', b'"col1","col2","col3"\n'), 144 | ('"', '"', b'col"1,col2,col3\n', b'"col""1","col2","col3"\n'), 145 | ('"', '"', b'""""\n', b'""""\n'), 146 | ('"', '"', b'\n', b''), 147 | ("'", "\\", b'col1,col2,col3\n', b"'col1','col2','col3'\n"), 148 | ("'", "\\", b'col""1,col2,col3\n', b"'col\"\"1','col2','col3'\n"), 149 | ("'", "\\", b'col\'1,col2,col3\n', b"'col\\'1','col2','col3'\n"), 150 | ("'", "\\", b'"col\'1","col2","col3"\n', b"'col\\'1','col2','col3'\n"), 151 | ("'", "\\", b'col\'\n', b"'col\\''\n"), 152 | # Two consecutive escaped quotes 153 | ("'", "\\", b'"a"""""\n', b"'a\"\"'\n"), 154 | ] 155 | 156 | client.make_bucket(bucket_name=bucket_name) 157 | 158 | try: 159 | for idx, (quote_char, escape_char, input_data, expected_output) in enumerate(tests): 160 | sql_opts = SelectRequest( 161 | "select * from s3object", 162 | CSVInputSerialization( 163 | compression_type=COMPRESSION_TYPE_NONE, 164 | file_header_info=FILE_HEADER_INFO_NONE, 165 | record_delimiter="\n", 166 | field_delimiter=",", 167 | quote_character='"', 168 | quote_escape_character='"', 169 | comments="#", 170 | allow_quoted_record_delimiter="FALSE", 171 | ), 172 | CSVOutputSerialization( 173 | quote_fields=QUOTE_FIELDS_ALWAYS, 174 | record_delimiter="\n", 175 | field_delimiter=",", 176 | quote_character=quote_char, 177 | quote_escape_character=escape_char, 178 | ), 179 | request_progress=False, 180 | ) 181 | 182 | test_sql_api(f'test_{idx}', client, bucket_name, 183 | input_data, sql_opts, expected_output) 184 | finally: 185 | client.remove_bucket(bucket_name=bucket_name) 186 | 187 | # Test passes 188 | print(log_output.json_report()) 189 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Mint [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/mint.svg?maxAge=604800)](https://hub.docker.com/r/minio/mint/) 2 | 3 | Mint is a testing framework for Minio object server, available as a podman image. It runs correctness, benchmarking and stress tests. Following are the SDKs/tools used in correctness tests. 4 | 5 | - awscli 6 | - aws-sdk-go-v2 7 | - aws-sdk-java-v2 8 | - aws-sdk-php 9 | - aws-sdk-ruby 10 | - healthcheck 11 | - mc 12 | - minio-go 13 | - minio-java 14 | - minio-js 15 | - minio-py 16 | - s3cmd 17 | - s3select 18 | - versioning 19 | 20 | ## Running Mint 21 | 22 | Mint is run by `podman run` command which requires Podman to be installed. For Podman installation follow the steps [here](https://podman.io/getting-started/installation#installing-on-linux). 23 | 24 | To run Mint with Minio Play server as test target, 25 | 26 | ```sh 27 | $ podman run -e SERVER_ENDPOINT=play.minio.io:9000 -e ACCESS_KEY=Q3AM3UQ867SPQQA43P2F \ 28 | -e SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG -e ENABLE_HTTPS=1 minio/mint 29 | ``` 30 | 31 | After the tests are run, output is stored in `/mint/log` directory inside the container. To get these logs, use `podman cp` command. For example 32 | ```sh 33 | podman cp :/mint/log /tmp/logs 34 | ``` 35 | 36 | ### Mint environment variables 37 | 38 | Below environment variables are required to be passed to the podman container. Supported environment variables: 39 | 40 | | Environment variable | Description | Example | 41 | |:-----------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------| 42 | | `SERVER_ENDPOINT` | Endpoint of Minio server in the format `HOST:PORT`; for virtual style `IP:PORT` | `play.minio.io:9000` | 43 | | `ACCESS_KEY` | Access key for `SERVER_ENDPOINT` credentials | `Q3AM3UQ867SPQQA43P2F` | 44 | | `SECRET_KEY` | Secret Key for `SERVER_ENDPOINT` credentials | `zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG` | 45 | | `ENABLE_HTTPS` | (Optional) Set `1` to indicate to use HTTPS to access `SERVER_ENDPOINT`. Defaults to `0` (HTTP) | `1` | 46 | | `MINT_MODE` | (Optional) Set mode indicating what category of tests to be run by values `core`, `full`. Defaults to `core` | `full` | 47 | | `DOMAIN` | (Optional) Value of MINIO_DOMAIN environment variable used in Minio server | `myminio.com` | 48 | | `ENABLE_VIRTUAL_STYLE` | (Optional) Set `1` to indicate virtual style access . Defaults to `0` (Path style) | `1` | 49 | | `RUN_ON_FAIL` | (Optional) Set `1` to indicate execute all tests independent of failures (currently implemented for minio-go and minio-java) . Defaults to `0` | `1` | 50 | | `SERVER_REGION` | (Optional) Set custom region for region specific tests | `us-west-1` | 51 | 52 | ### Test virtual style access against Minio server 53 | 54 | To test Minio server virtual style access with Mint, follow these steps: 55 | 56 | - Set a domain in your Minio server using environment variable MINIO_DOMAIN. For example `export MINIO_DOMAIN=myminio.com`. 57 | - Start Minio server. 58 | - Execute Mint against Minio server (with `MINIO_DOMAIN` set to `myminio.com`) using this command 59 | ```sh 60 | $ podman run -e "SERVER_ENDPOINT=192.168.86.133:9000" -e "DOMAIN=minio.com" \ 61 | -e "ACCESS_KEY=minio" -e "SECRET_KEY=minio123" -e "ENABLE_HTTPS=0" \ 62 | -e "ENABLE_VIRTUAL_STYLE=1" minio/mint 63 | ``` 64 | 65 | ### Mint log format 66 | 67 | All test logs are stored in `/mint/log/log.json` as multiple JSON document. Below is the JSON format for every entry in the log file. 68 | 69 | | JSON field | Type | Description | Example | 70 | |:-----------|:---------|:--------------------------------------------------------------|:------------------------------------------------------| 71 | | `name` | _string_ | Testing tool/SDK name | `"aws-sdk-php"` | 72 | | `function` | _string_ | Test function name | `"getBucketLocation ( array $params = [] )"` | 73 | | `args` | _object_ | (Optional) Key/Value map of arguments passed to test function | `{"Bucket":"aws-sdk-php-bucket-20341"}` | 74 | | `duration` | _int_ | Time taken in milliseconds to run the test | `384` | 75 | | `status` | _string_ | one of `PASS`, `FAIL` or `NA` | `"PASS"` | 76 | | `alert` | _string_ | (Optional) Alert message indicating test failure | `"I/O error on create file"` | 77 | | `message` | _string_ | (Optional) Any log message | `"validating checksum of downloaded object"` | 78 | | `error` | _string_ | Detailed error message including stack trace on status `FAIL` | `"Error executing \"CompleteMultipartUpload\" on ...` | 79 | 80 | ## For Developers 81 | 82 | ### Running Mint development code 83 | 84 | After making changes to Mint source code a local podman image can be built/run by 85 | 86 | ```sh 87 | $ podman build -t minio/mint . -f Dockerfile 88 | $ podman run -e SERVER_ENDPOINT=play.minio.io:9000 -e ACCESS_KEY=Q3AM3UQ867SPQQA43P2F \ 89 | -e SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG \ 90 | -e ENABLE_HTTPS=1 -e MINT_MODE=full minio/mint:latest 91 | ``` 92 | 93 | 94 | ### Adding tests with new tool/SDK 95 | 96 | Below are the steps need to be followed 97 | 98 | - Create new app directory under [build](https://github.com/minio/mint/tree/master/build) and [run/core](https://github.com/minio/mint/tree/master/run/core) directories. 99 | - Create `install.sh` which does installation of required tool/SDK under app directory. 100 | - Any build and install time dependencies should be added to [install-packages.list](https://github.com/minio/mint/blob/master/install-packages.list). 101 | - Build time dependencies should be added to [remove-packages.list](https://github.com/minio/mint/blob/master/remove-packages.list) for removal to have clean Mint podman image. 102 | - Add `run.sh` in app directory under `run/core` which execute actual tests. 103 | 104 | #### Test data 105 | Tests may use pre-created data set to perform various object operations on Minio server. Below data files are available under `/mint/data` directory. 106 | 107 | | File name | Size | 108 | |:-----------------|:--------| 109 | | datafile-0-b | 0B | 110 | | datafile-1-b | 1B | 111 | | datafile-1-kB | 1KiB | 112 | | datafile-10-kB | 10KiB | 113 | | datafile-33-kB | 33KiB | 114 | | datafile-100-kB | 100KiB | 115 | | datafile-1-MB | 1MiB | 116 | | datafile-1.03-MB | 1.03MiB | 117 | | datafile-5-MB | 5MiB | 118 | | datafile-6-MB | 6MiB | 119 | | datafile-10-MB | 10MiB | 120 | | datafile-11-MB | 11MiB | 121 | | datafile-65-MB | 65MiB | 122 | | datafile-129-MB | 129MiB | 123 | 124 | ### Updating SDKs/binaries in the image 125 | 126 | In many cases, updating the SDKs or binaries in the image is just a matter of making a commit updating the corresponding version in this repo. However, in some cases, e.g. when `mc` needs to be updated (the latest `mc` is pulled in during each mint image build), a sort of "dummy" commit is required as an image rebuild must be triggered. Note that an empty commit does not appear to trigger the image rebuild in the Docker Hub. 127 | -------------------------------------------------------------------------------- /run/core/healthcheck/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Mint, (C) 2019 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | package main 21 | 22 | import ( 23 | "crypto/tls" 24 | "encoding/json" 25 | "fmt" 26 | "net/http" 27 | "net/url" 28 | "os" 29 | "time" 30 | 31 | jwtgo "github.com/golang-jwt/jwt/v4" 32 | log "github.com/sirupsen/logrus" 33 | ) 34 | 35 | const ( 36 | pass = "PASS" // Indicate that a test passed 37 | fail = "FAIL" // Indicate that a test failed 38 | livenessPath = "/minio/health/live" 39 | readinessPath = "/minio/health/ready" 40 | prometheusPathV2Cluster = "/minio/v2/metrics/cluster" 41 | prometheusPathV2Node = "/minio/v2/metrics/node" 42 | prometheusPathV2Bucket = "/minio/v2/metrics/bucket" 43 | prometheusPathV2Resource = "/minio/v2/metrics/resource" 44 | timeout = time.Duration(30 * time.Second) 45 | ) 46 | 47 | type mintJSONFormatter struct{} 48 | 49 | func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) { 50 | data := make(log.Fields, len(entry.Data)) 51 | for k, v := range entry.Data { 52 | switch v := v.(type) { 53 | case error: 54 | // Otherwise errors are ignored by `encoding/json` 55 | // https://github.com/sirupsen/logrus/issues/137 56 | data[k] = v.Error() 57 | default: 58 | data[k] = v 59 | } 60 | } 61 | 62 | serialized, err := json.Marshal(data) 63 | if err != nil { 64 | return nil, fmt.Errorf("Failed to marshal fields to JSON, %w", err) 65 | } 66 | return append(serialized, '\n'), nil 67 | } 68 | 69 | // log successful test runs 70 | func successLogger(function string, args map[string]interface{}, startTime time.Time) *log.Entry { 71 | // calculate the test case duration 72 | duration := time.Since(startTime) 73 | // log with the fields as per mint 74 | fields := log.Fields{"name": "healthcheck", "function": function, "args": args, "duration": duration.Nanoseconds() / 1000000, "status": pass} 75 | return log.WithFields(fields) 76 | } 77 | 78 | // log failed test runs 79 | func failureLog(function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) *log.Entry { 80 | // calculate the test case duration 81 | duration := time.Since(startTime) 82 | var fields log.Fields 83 | // log with the fields as per mint 84 | if err != nil { 85 | fields = log.Fields{ 86 | "name": "healthcheck", "function": function, "args": args, 87 | "duration": duration.Nanoseconds() / 1000000, "status": fail, "alert": alert, "message": message, "error": err, 88 | } 89 | } else { 90 | fields = log.Fields{ 91 | "name": "healthcheck", "function": function, "args": args, 92 | "duration": duration.Nanoseconds() / 1000000, "status": fail, "alert": alert, "message": message, 93 | } 94 | } 95 | return log.WithFields(fields) 96 | } 97 | 98 | func testLivenessEndpoint(endpoint string) { 99 | startTime := time.Now() 100 | function := "testLivenessEndpoint" 101 | 102 | u, err := url.Parse(fmt.Sprintf("%s%s", endpoint, livenessPath)) 103 | if err != nil { 104 | // Could not parse URL successfully 105 | failureLog(function, nil, startTime, "", "URL Parsing for Healthcheck Liveness handler failed", err).Fatal() 106 | } 107 | 108 | tr := &http.Transport{ 109 | TLSClientConfig: &tls.Config{InsecureSkipVerify: u.Scheme == "https"}, 110 | } 111 | client := &http.Client{Transport: tr, Timeout: timeout} 112 | resp, err := client.Get(u.String()) 113 | if err != nil { 114 | // GET request errored 115 | failureLog(function, nil, startTime, "", "GET request failed", err).Fatal() 116 | } 117 | if resp.StatusCode != http.StatusOK { 118 | // Status not 200 OK 119 | failureLog(function, nil, startTime, "", fmt.Sprintf("GET /minio/health/live returned %s", resp.Status), err).Fatal() 120 | } 121 | 122 | defer resp.Body.Close() 123 | defer successLogger(function, nil, startTime).Info() 124 | } 125 | 126 | func testReadinessEndpoint(endpoint string) { 127 | startTime := time.Now() 128 | function := "testReadinessEndpoint" 129 | 130 | u, err := url.Parse(fmt.Sprintf("%s%s", endpoint, readinessPath)) 131 | if err != nil { 132 | // Could not parse URL successfully 133 | failureLog(function, nil, startTime, "", "URL Parsing for Healthcheck Readiness handler failed", err).Fatal() 134 | } 135 | 136 | tr := &http.Transport{ 137 | TLSClientConfig: &tls.Config{InsecureSkipVerify: u.Scheme == "https"}, 138 | } 139 | client := &http.Client{Transport: tr, Timeout: timeout} 140 | resp, err := client.Get(u.String()) 141 | if err != nil { 142 | // GET request errored 143 | failureLog(function, nil, startTime, "", "GET request to Readiness endpoint failed", err).Fatal() 144 | } 145 | if resp.StatusCode != http.StatusOK { 146 | // Status not 200 OK 147 | failureLog(function, nil, startTime, "", "GET /minio/health/ready returned non OK status", err).Fatal() 148 | } 149 | 150 | defer resp.Body.Close() 151 | defer successLogger(function, nil, startTime).Info() 152 | } 153 | 154 | const ( 155 | defaultPrometheusJWTExpiry = 100 * 365 * 24 * time.Hour 156 | ) 157 | 158 | func testPrometheusEndpointV2(endpoint string, metricsPath string) { 159 | startTime := time.Now() 160 | function := "testPrometheusEndpoint" 161 | 162 | u, err := url.Parse(fmt.Sprintf("%s%s", endpoint, metricsPath)) 163 | if err != nil { 164 | // Could not parse URL successfully 165 | failureLog(function, nil, startTime, "", "URL Parsing for Healthcheck Prometheus handler failed", err).Fatal() 166 | } 167 | 168 | jwt := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, jwtgo.StandardClaims{ 169 | ExpiresAt: time.Now().UTC().Add(defaultPrometheusJWTExpiry).Unix(), 170 | Subject: os.Getenv("ACCESS_KEY"), 171 | Issuer: "prometheus", 172 | }) 173 | 174 | token, err := jwt.SignedString([]byte(os.Getenv("SECRET_KEY"))) 175 | if err != nil { 176 | failureLog(function, nil, startTime, "", "jwt generation failed", err).Fatal() 177 | } 178 | 179 | tr := &http.Transport{ 180 | TLSClientConfig: &tls.Config{InsecureSkipVerify: u.Scheme == "https"}, 181 | } 182 | client := &http.Client{Transport: tr, Timeout: timeout} 183 | 184 | req, err := http.NewRequest(http.MethodGet, u.String(), nil) 185 | if err != nil { 186 | failureLog(function, nil, startTime, "", "Initializing GET request to Prometheus endpoint failed", err).Fatal() 187 | } 188 | req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) 189 | 190 | resp, err := client.Do(req) 191 | if err != nil { 192 | // GET request errored 193 | failureLog(function, nil, startTime, "", "GET request to Prometheus endpoint failed", err).Fatal() 194 | } 195 | 196 | if resp.StatusCode != http.StatusOK { 197 | // Status not 200 OK 198 | failureLog(function, nil, startTime, "", "GET "+endpoint+" returned non OK status", err).Fatal() 199 | } 200 | 201 | defer resp.Body.Close() 202 | defer successLogger(function, nil, startTime).Info() 203 | } 204 | 205 | func testClusterPrometheusEndpointV2(endpoint string) { 206 | testPrometheusEndpointV2(endpoint, prometheusPathV2Cluster) 207 | } 208 | 209 | func testNodePrometheusEndpointV2(endpoint string) { 210 | testPrometheusEndpointV2(endpoint, prometheusPathV2Node) 211 | } 212 | 213 | func testBucketPrometheusEndpointV2(endpoint string) { 214 | testPrometheusEndpointV2(endpoint, prometheusPathV2Bucket) 215 | } 216 | 217 | func testResourcePrometheusEndpointV2(endpoint string) { 218 | testPrometheusEndpointV2(endpoint, prometheusPathV2Resource) 219 | } 220 | 221 | func main() { 222 | endpoint := os.Getenv("SERVER_ENDPOINT") 223 | secure := os.Getenv("ENABLE_HTTPS") 224 | if secure == "1" { 225 | endpoint = "https://" + endpoint 226 | } else { 227 | endpoint = "http://" + endpoint 228 | } 229 | 230 | // Output to stdout instead of the default stderr 231 | log.SetOutput(os.Stdout) 232 | // create custom formatter 233 | mintFormatter := mintJSONFormatter{} 234 | // set custom formatter 235 | log.SetFormatter(&mintFormatter) 236 | // log Info or above -- success cases are Info level, failures are Fatal level 237 | log.SetLevel(log.InfoLevel) 238 | // execute tests 239 | testLivenessEndpoint(endpoint) 240 | testReadinessEndpoint(endpoint) 241 | testClusterPrometheusEndpointV2(endpoint) 242 | testNodePrometheusEndpointV2(endpoint) 243 | testBucketPrometheusEndpointV2(endpoint) 244 | testResourcePrometheusEndpointV2(endpoint) 245 | } 246 | -------------------------------------------------------------------------------- /run/core/s3cmd/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Mint (C) 2017-2020 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # shellcheck disable=SC2317 19 | 20 | if [ -n "$MINT_MODE" ]; then 21 | if [ -z "${MINT_DATA_DIR+x}" ]; then 22 | echo "MINT_DATA_DIR not defined" 23 | exit 1 24 | fi 25 | if [ -z "${SERVER_ENDPOINT+x}" ]; then 26 | echo "SERVER_ENDPOINT not defined" 27 | exit 1 28 | fi 29 | if [ -z "${ACCESS_KEY+x}" ]; then 30 | echo "ACCESS_KEY not defined" 31 | exit 1 32 | fi 33 | if [ -z "${SECRET_KEY+x}" ]; then 34 | echo "SECRET_KEY not defined" 35 | exit 1 36 | fi 37 | fi 38 | 39 | if [ -z "${SERVER_ENDPOINT+x}" ]; then 40 | SERVER_ENDPOINT="play.minio.io:9000" 41 | ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" 42 | SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" 43 | ENABLE_HTTPS=1 44 | SERVER_REGION="us-east-1" 45 | fi 46 | 47 | WORK_DIR="$PWD" 48 | DATA_DIR="$MINT_DATA_DIR" 49 | if [ -z "$MINT_MODE" ]; then 50 | WORK_DIR="$PWD/.run-$RANDOM" 51 | DATA_DIR="$WORK_DIR/data" 52 | fi 53 | 54 | FILE_1_MB="$DATA_DIR/datafile-1-MB" 55 | FILE_65_MB="$DATA_DIR/datafile-65-MB" 56 | declare FILE_1_MB_MD5SUM 57 | declare FILE_65_MB_MD5SUM 58 | 59 | BUCKET_NAME="s3cmd-test-bucket-$RANDOM" 60 | S3CMD=$(command -v s3cmd) 61 | declare -a S3CMD_CMD 62 | 63 | function get_md5sum() { 64 | filename="$FILE_1_MB" 65 | out=$(md5sum "$filename" 2>/dev/null) 66 | rv=$? 67 | if [ "$rv" -eq 0 ]; then 68 | awk '{ print $1 }' <<<"$out" 69 | fi 70 | 71 | return "$rv" 72 | } 73 | 74 | function get_time() { 75 | date +%s%N 76 | } 77 | 78 | function get_duration() { 79 | start_time=$1 80 | end_time=$(get_time) 81 | 82 | echo $(((end_time - start_time) / 1000000)) 83 | } 84 | 85 | function log_success() { 86 | if [ -n "$MINT_MODE" ]; then 87 | printf '{"name": "s3cmd", "duration": "%d", "function": "%s", "status": "PASS"}\n' "$(get_duration "$1")" "$2" 88 | fi 89 | } 90 | 91 | function show() { 92 | if [ -z "$MINT_MODE" ]; then 93 | func_name="$1" 94 | echo "Running $func_name()" 95 | fi 96 | } 97 | 98 | function fail() { 99 | rv="$1" 100 | shift 101 | 102 | if [ "$rv" -ne 0 ]; then 103 | echo "$@" 104 | fi 105 | 106 | return "$rv" 107 | } 108 | 109 | function assert() { 110 | expected_rv="$1" 111 | shift 112 | start_time="$1" 113 | shift 114 | func_name="$1" 115 | shift 116 | 117 | err=$("$@" 2>&1) 118 | rv=$? 119 | if [ "$rv" -ne 0 ] && [ "$expected_rv" -eq 0 ]; then 120 | if [ -n "$MINT_MODE" ]; then 121 | err=$(printf '%s' "$err" | python -c 'import sys,json; print(json.dumps(sys.stdin.read()))') 122 | ## err is already JSON string, no need to double quote 123 | printf '{"name": "s3cmd", "duration": "%d", "function": "%s", "status": "FAIL", "error": %s}\n' "$(get_duration "$start_time")" "$func_name" "$err" 124 | else 125 | echo "s3cmd: $func_name: $err" 126 | fi 127 | 128 | exit "$rv" 129 | fi 130 | 131 | return 0 132 | } 133 | 134 | function assert_success() { 135 | assert 0 "$@" 136 | } 137 | 138 | function assert_failure() { 139 | assert 1 "$@" 140 | } 141 | 142 | function s3cmd_cmd() { 143 | cmd=("${S3CMD_CMD[@]}" "$@") 144 | "${cmd[@]}" 145 | rv=$? 146 | return "$rv" 147 | } 148 | 149 | function check_md5sum() { 150 | expected_checksum="$1" 151 | shift 152 | filename="$*" 153 | 154 | checksum="$(get_md5sum "$filename")" 155 | rv=$? 156 | if [ "$rv" -ne 0 ]; then 157 | echo "unable to get md5sum for $filename" 158 | return "$rv" 159 | fi 160 | 161 | if [ "$checksum" != "$expected_checksum" ]; then 162 | echo "$filename: md5sum mismatch" 163 | return 1 164 | fi 165 | 166 | return 0 167 | } 168 | 169 | function test_make_bucket() { 170 | show "${FUNCNAME[0]}" 171 | 172 | start_time=$(get_time) 173 | bucket_name="s3cmd-test-bucket-$RANDOM" 174 | assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${bucket_name}" 175 | assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rb "s3://${bucket_name}" 176 | 177 | log_success "$start_time" "${FUNCNAME[0]}" 178 | } 179 | 180 | function test_make_bucket_error() { 181 | show "${FUNCNAME[0]}" 182 | 183 | start_time=$(get_time) 184 | bucket_name="S3CMD-test%bucket%$RANDOM" 185 | assert_failure "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${bucket_name}" 186 | 187 | log_success "$start_time" "${FUNCNAME[0]}" 188 | } 189 | 190 | function setup() { 191 | start_time=$(get_time) 192 | assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${BUCKET_NAME}" 193 | } 194 | 195 | function teardown() { 196 | start_time=$(get_time) 197 | assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm --force --recursive "s3://${BUCKET_NAME}" 198 | assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rb --force "s3://${BUCKET_NAME}" 199 | } 200 | 201 | function test_put_object() { 202 | show "${FUNCNAME[0]}" 203 | 204 | start_time=$(get_time) 205 | object_name="s3cmd-test-object-$RANDOM" 206 | assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name}" 207 | assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}" 208 | 209 | log_success "$start_time" "${FUNCNAME[0]}" 210 | } 211 | 212 | function test_put_object_error() { 213 | show "${FUNCNAME[0]}" 214 | start_time=$(get_time) 215 | 216 | object_long_name=$(printf "s3cmd-test-object-%01100d" 1) 217 | assert_failure "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_long_name}" 218 | 219 | log_success "$start_time" "${FUNCNAME[0]}" 220 | } 221 | 222 | function test_put_object_multipart() { 223 | show "${FUNCNAME[0]}" 224 | 225 | start_time=$(get_time) 226 | object_name="s3cmd-test-object-$RANDOM" 227 | assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_65_MB}" "s3://${BUCKET_NAME}/${object_name}" 228 | assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}" 229 | 230 | log_success "$start_time" "${FUNCNAME[0]}" 231 | } 232 | 233 | function test_get_object() { 234 | show "${FUNCNAME[0]}" 235 | 236 | start_time=$(get_time) 237 | object_name="s3cmd-test-object-$RANDOM" 238 | assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name}" 239 | assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd get "s3://${BUCKET_NAME}/${object_name}" "${object_name}.downloaded" 240 | assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_1_MB_MD5SUM" "${object_name}.downloaded" 241 | assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}" 242 | assert_success "$start_time" "${FUNCNAME[0]}" rm -f "${object_name}.downloaded" 243 | 244 | log_success "$start_time" "${FUNCNAME[0]}" 245 | } 246 | 247 | function test_get_object_error() { 248 | show "${FUNCNAME[0]}" 249 | 250 | start_time=$(get_time) 251 | object_name="s3cmd-test-object-$RANDOM" 252 | assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name}" 253 | assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}" 254 | assert_failure "$start_time" "${FUNCNAME[0]}" s3cmd_cmd get "s3://${BUCKET_NAME}/${object_name}" "${object_name}.downloaded" 255 | assert_success "$start_time" "${FUNCNAME[0]}" rm -f "${object_name}.downloaded" 256 | 257 | log_success "$start_time" "${FUNCNAME[0]}" 258 | } 259 | 260 | function test_get_object_multipart() { 261 | show "${FUNCNAME[0]}" 262 | 263 | start_time=$(get_time) 264 | object_name="s3cmd-test-object-$RANDOM" 265 | assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_65_MB}" "s3://${BUCKET_NAME}/${object_name}" 266 | assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd get "s3://${BUCKET_NAME}/${object_name}" "${object_name}.downloaded" 267 | assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_65_MB_MD5SUM" "${object_name}.downloaded" 268 | assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}" 269 | assert_success "$start_time" "${FUNCNAME[0]}" rm -f "${object_name}.downloaded" 270 | 271 | log_success "$start_time" "${FUNCNAME[0]}" 272 | } 273 | 274 | function test_sync_list_objects() { 275 | show "${FUNCNAME[0]}" 276 | 277 | start_time=$(get_time) 278 | bucket_name="s3cmd-test-bucket-$RANDOM" 279 | object_name="s3cmd-test-object-$RANDOM" 280 | assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${bucket_name}" 281 | assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd sync "$DATA_DIR/" "s3://${bucket_name}" 282 | 283 | diff -bB <(ls "$DATA_DIR") <("${S3CMD_CMD[@]}" ls "s3://${bucket_name}" | awk '{print $4}' | sed "s/s3:*..${bucket_name}.//g") >/dev/null 2>&1 284 | assert_success "$start_time" "${FUNCNAME[0]}" fail $? "sync and list differs" 285 | assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rb --force --recursive "s3://${bucket_name}" 286 | 287 | log_success "$start_time" "${FUNCNAME[0]}" 288 | } 289 | 290 | function run_test() { 291 | test_make_bucket 292 | test_make_bucket_error 293 | 294 | setup 295 | 296 | test_put_object 297 | test_put_object_error 298 | test_put_object_multipart 299 | test_get_object 300 | test_get_object_multipart 301 | test_sync_list_objects 302 | 303 | teardown 304 | } 305 | 306 | function __init__() { 307 | set -e 308 | 309 | S3CMD_CONFIG_DIR="/tmp/.s3cmd-$RANDOM" 310 | mkdir -p $S3CMD_CONFIG_DIR 311 | S3CMD_CONFIG_FILE="$S3CMD_CONFIG_DIR/s3cfg" 312 | 313 | # configure s3cmd 314 | cat >$S3CMD_CONFIG_FILE <"$FILE_1_MB" 345 | fi 346 | 347 | if [ ! -e "$FILE_65_MB" ]; then 348 | shred -n 1 -s 65MB - >"$FILE_65_MB" 349 | fi 350 | 351 | set -E 352 | set -o pipefail 353 | 354 | FILE_1_MB_MD5SUM="$(get_md5sum "$FILE_1_MB")" 355 | rv=$? 356 | if [ $rv -ne 0 ]; then 357 | echo "unable to get md5sum of $FILE_1_MB" 358 | exit 1 359 | fi 360 | 361 | FILE_65_MB_MD5SUM="$(get_md5sum "$FILE_65_MB")" 362 | rv=$? 363 | if [ $rv -ne 0 ]; then 364 | echo "unable to get md5sum of $FILE_65_MB" 365 | exit 1 366 | fi 367 | 368 | set +e 369 | } 370 | 371 | function main() { 372 | (run_test) 373 | rv=$? 374 | 375 | rm -fr "$S3CMD_CONFIG_FILE" 376 | if [ -z "$MINT_MODE" ]; then 377 | rm -fr "$WORK_DIR" "$DATA_DIR" 378 | fi 379 | 380 | exit "$rv" 381 | } 382 | 383 | __init__ "$@" 384 | main "$@" 385 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /run/core/s3select/sql_ops.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # MinIO Python Library for Amazon S3 Compatible Cloud Storage, 4 | # (C) 2020 MinIO, Inc. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | import io 19 | from datetime import datetime 20 | 21 | from minio.select import (FILE_HEADER_INFO_NONE, JSON_TYPE_DOCUMENT, 22 | QUOTE_FIELDS_ASNEEDED, CSVInputSerialization, 23 | CSVOutputSerialization, JSONInputSerialization, 24 | JSONOutputSerialization, SelectRequest) 25 | 26 | from utils import generate_bucket_name, generate_object_name 27 | 28 | 29 | def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, 30 | sql_output, tests, log_output): 31 | bucket_name = generate_bucket_name() 32 | object_name = generate_object_name() 33 | 34 | log_output.args['total_tests'] = 0 35 | log_output.args['total_success'] = 0 36 | 37 | client.make_bucket(bucket_name=bucket_name) 38 | try: 39 | content = io.BytesIO(bytes(input_bytes, 'utf-8')) 40 | client.put_object( 41 | bucket_name=bucket_name, 42 | object_name=object_name, 43 | data=content, 44 | length=len(input_bytes) 45 | ) 46 | 47 | for idx, (test_name, select_expression, expected_output) in enumerate(tests): 48 | if select_expression == '': 49 | continue 50 | try: 51 | log_output.args['total_tests'] += 1 52 | sreq = SelectRequest( 53 | select_expression, 54 | sql_input, 55 | sql_output, 56 | request_progress=False 57 | ) 58 | 59 | data = client.select_object_content( 60 | bucket_name=bucket_name, 61 | object_name=object_name, 62 | request=sreq 63 | ) 64 | 65 | # Get the records 66 | records = io.BytesIO() 67 | for d in data.stream(10*1024): 68 | records.write(d) 69 | got_output = records.getvalue() 70 | 71 | if got_output != expected_output: 72 | if type(expected_output) == datetime: 73 | # Attempt to parse the date which will throw an exception for any issue 74 | datetime.strptime(got_output.decode( 75 | "utf-8").strip(), '%Y-%m-%dT%H:%M:%S.%f%z') 76 | else: 77 | raise ValueError('Test {}: data mismatch. Expected : {}. Received: {}.'.format( 78 | idx+1, expected_output, got_output)) 79 | 80 | log_output.args['total_success'] += 1 81 | except Exception as err: 82 | continue # TODO, raise instead 83 | # raise Exception(err) 84 | finally: 85 | client.remove_object(bucket_name=bucket_name, object_name=object_name) 86 | client.remove_bucket(bucket_name=bucket_name) 87 | 88 | 89 | def test_sql_expressions(client, input_json_bytes, tests, log_output): 90 | input_serialization = JSONInputSerialization( 91 | compression_type="NONE", 92 | json_type=JSON_TYPE_DOCUMENT, 93 | ) 94 | 95 | output_serialization = CSVOutputSerialization( 96 | quote_fields=QUOTE_FIELDS_ASNEEDED) 97 | 98 | test_sql_expressions_custom_input_output(client, input_json_bytes, 99 | input_serialization, output_serialization, tests, log_output) 100 | 101 | 102 | def test_sql_operators(client, log_output): 103 | 104 | json_testfile = """{"id": 1, "name": "John", "age": 3} 105 | {"id": 2, "name": "Elliot", "age": 4} 106 | {"id": 3, "name": "Yves", "age": 5} 107 | {"id": 4, "name": null, "age": 0} 108 | """ 109 | 110 | tests = [ 111 | # Logical operators 112 | ("AND", "select * from S3Object s where s.id = 1 AND s.name = 'John'", b'1,John,3\n'), 113 | ("NOT", "select * from S3Object s where NOT s.id = 1", 114 | b'2,Elliot,4\n3,Yves,5\n4,,0\n'), 115 | ("OR", "select * from S3Object s where s.id = 1 OR s.id = 3", 116 | b'1,John,3\n3,Yves,5\n'), 117 | # Comparison Operators 118 | ("<", "select * from S3Object s where s.age < 4", b'1,John,3\n4,,0\n'), 119 | (">", "select * from S3Object s where s.age > 4", b'3,Yves,5\n'), 120 | ("<=", "select * from S3Object s where s.age <= 4", 121 | b'1,John,3\n2,Elliot,4\n4,,0\n'), 122 | (">=", "select * from S3Object s where s.age >= 4", b'2,Elliot,4\n3,Yves,5\n'), 123 | ("=", "select * from S3Object s where s.age = 4", b'2,Elliot,4\n'), 124 | ("<>", "select * from S3Object s where s.age <> 4", 125 | b'1,John,3\n3,Yves,5\n4,,0\n'), 126 | ("!=", "select * from S3Object s where s.age != 4", 127 | b'1,John,3\n3,Yves,5\n4,,0\n'), 128 | ("BETWEEN", "select * from S3Object s where s.age BETWEEN 4 AND 5", 129 | b'2,Elliot,4\n3,Yves,5\n'), 130 | ("IN", "select * from S3Object s where s.age IN (3,5)", b'1,John,3\n3,Yves,5\n'), 131 | # Pattern Matching Operators 132 | ("LIKE_", "select * from S3Object s where s.name LIKE '_ves'", b'3,Yves,5\n'), 133 | ("LIKE%", "select * from S3Object s where s.name LIKE 'Ell%t'", b'2,Elliot,4\n'), 134 | # Unitary Operators 135 | ("NULL", "select * from S3Object s where s.name IS NULL", b'4,,0\n'), 136 | ("NOT_NULL", "select * from S3Object s where s.age IS NOT NULL", 137 | b'1,John,3\n2,Elliot,4\n3,Yves,5\n4,,0\n'), 138 | # Math Operators 139 | ("+", "select * from S3Object s where s.age = 1+3 ", b'2,Elliot,4\n'), 140 | ("-", "select * from S3Object s where s.age = 5-1 ", b'2,Elliot,4\n'), 141 | ("*", "select * from S3Object s where s.age = 2*2 ", b'2,Elliot,4\n'), 142 | ("%", "select * from S3Object s where s.age = 10%6 ", b'2,Elliot,4\n'), 143 | ] 144 | 145 | try: 146 | test_sql_expressions(client, json_testfile, tests, log_output) 147 | except Exception as select_err: 148 | raise select_err 149 | # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) 150 | # pass 151 | 152 | # Test passes 153 | print(log_output.json_report()) 154 | 155 | 156 | def test_sql_operators_precedence(client, log_output): 157 | 158 | json_testfile = """{"id": 1, "name": "Eric"}""" 159 | 160 | tests = [ 161 | ("-_1", "select -3*3 from S3Object", b'-9\n'), 162 | ("*", "select 10-3*2 from S3Object", b'4\n'), 163 | ("/", "select 13-10/5 from S3Object", b'11\n'), 164 | ("%", "select 13-10%5 from S3Object", b'13\n'), 165 | ("+", "select 1+1*3 from S3Object", b'4\n'), 166 | ("-_2", "select 1-1*3 from S3Object", b'-2\n'), 167 | ("=", "select * from S3Object as s where s.id = 13-12", b'1,Eric\n'), 168 | ("<>", "select * from S3Object as s where s.id <> 1-1", b'1,Eric\n'), 169 | ("NOT", "select * from S3Object where false OR NOT false", b'1,Eric\n'), 170 | ("AND", "select * from S3Object where true AND true OR false ", b'1,Eric\n'), 171 | ("OR", "select * from S3Object where false OR NOT false", b'1,Eric\n'), 172 | ("IN", "select * from S3Object as s where s.id <> -1 AND s.id IN (1,2,3)", b'1,Eric\n'), 173 | ("BETWEEN", "select * from S3Object as s where s.id <> -1 AND s.id BETWEEN -1 AND 3", b'1,Eric\n'), 174 | ("LIKE", "select * from S3Object as s where s.id <> -1 AND s.name LIKE 'E%'", b'1,Eric\n'), 175 | ] 176 | 177 | try: 178 | test_sql_expressions(client, json_testfile, tests, log_output) 179 | except Exception as select_err: 180 | raise select_err 181 | # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) 182 | # pass 183 | 184 | # Test passes 185 | print(log_output.json_report()) 186 | 187 | 188 | def test_sql_functions_agg_cond_conv(client, log_output): 189 | 190 | json_testfile = """{"id": 1, "name": "John", "age": 3} 191 | {"id": 2, "name": "Elliot", "age": 4} 192 | {"id": 3, "name": "Yves", "age": 5} 193 | {"id": 4, "name": "Christine", "age": null} 194 | {"id": 5, "name": "Eric", "age": 0} 195 | """ 196 | tests = [ 197 | # Aggregate functions 198 | ("COUNT", "select count(*) from S3Object s", b'5\n'), 199 | ("AVG", "select avg(s.age) from S3Object s", b'3\n'), 200 | ("MAX", "select max(s.age) from S3Object s", b'5\n'), 201 | ("MIN", "select min(s.age) from S3Object s", b'0\n'), 202 | ("SUM", "select sum(s.age) from S3Object s", b'12\n'), 203 | # Conditional functions 204 | ("COALESCE", "SELECT COALESCE(s.age, 99) FROM S3Object s", b'3\n4\n5\n99\n0\n'), 205 | ("NULLIF", "SELECT NULLIF(s.age, 0) FROM S3Object s", b'3\n4\n5\n\n\n'), 206 | # Conversion functions 207 | ("CAST", "SELECT CAST(s.age AS FLOAT) FROM S3Object s", 208 | b'3.0\n4.0\n5.0\n\n0.0\n'), 209 | 210 | ] 211 | 212 | try: 213 | test_sql_expressions(client, json_testfile, tests, log_output) 214 | except Exception as select_err: 215 | raise select_err 216 | # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) 217 | # pass 218 | 219 | # Test passes 220 | print(log_output.json_report()) 221 | 222 | 223 | def test_sql_functions_date(client, log_output): 224 | 225 | json_testfile = """ 226 | {"id": 1, "name": "John", "datez": "2017-01-02T03:04:05.006+07:30"} 227 | """ 228 | 229 | tests = [ 230 | # DATE_ADD 231 | ("DATE_ADD_1", "select DATE_ADD(year, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", 232 | b'2022-01-02T03:04:05.006+07:30\n'), 233 | ("DATE_ADD_2", "select DATE_ADD(month, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", 234 | b'2017-02-02T03:04:05.006+07:30\n'), 235 | ("DATE_ADD_3", "select DATE_ADD(day, -1, TO_TIMESTAMP(s.datez)) from S3Object as s", 236 | b'2017-01-01T03:04:05.006+07:30\n'), 237 | ("DATE_ADD_4", "select DATE_ADD(hour, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", 238 | b'2017-01-02T04:04:05.006+07:30\n'), 239 | ("DATE_ADD_5", "select DATE_ADD(minute, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", 240 | b'2017-01-02T03:09:05.006+07:30\n'), 241 | ("DATE_ADD_6", "select DATE_ADD(second, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", 242 | b'2017-01-02T03:04:10.006+07:30\n'), 243 | # DATE_DIFF 244 | ("DATE_DIFF_1", "select DATE_DIFF(year, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011-01-01T')) from S3Object as s", b'-6\n'), 245 | ("DATE_DIFF_2", "select DATE_DIFF(month, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011T')) from S3Object as s", b'-72\n'), 246 | ("DATE_DIFF_3", "select DATE_DIFF(day, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2010-01-02T')) from S3Object as s", b'-2556\n'), 247 | # EXTRACT 248 | ("EXTRACT_1", "select EXTRACT(year FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017\n'), 249 | ("EXTRACT_2", "select EXTRACT(month FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'1\n'), 250 | ("EXTRACT_3", "select EXTRACT(hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'3\n'), 251 | ("EXTRACT_4", "select EXTRACT(minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'4\n'), 252 | ("EXTRACT_5", "select EXTRACT(timezone_hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'7\n'), 253 | ("EXTRACT_6", "select EXTRACT(timezone_minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'30\n'), 254 | # TO_STRING 255 | ("TO_STRING_1", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y') from S3Object as s", 256 | b'"January 2, 2017"\n'), 257 | ("TO_STRING_2", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMM d, yyyy') from S3Object as s", b'"Jan 2, 2017"\n'), 258 | ("TO_STRING_3", "select TO_STRING(TO_TIMESTAMP(s.datez), 'M-d-yy') from S3Object as s", b'1-2-17\n'), 259 | ("TO_STRING_4", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MM-d-y') from S3Object as s", b'01-2-2017\n'), 260 | ("TO_STRING_5", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y h:m a') from S3Object as s", 261 | b'"January 2, 2017 3:4 AM"\n'), 262 | ("TO_STRING_6", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", 263 | b'2017-01-02T3:4:05+0730\n'), 264 | ("TO_STRING_7", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", 265 | b'2017-01-02T3:4:05+0730\n'), 266 | ("TO_STRING_8", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXX') from S3Object as s", 267 | b'2017-01-02T3:4:05+0730\n'), 268 | ("TO_STRING_9", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXXX') from S3Object as s", 269 | b'2017-01-02T3:4:05+07:30\n'), 270 | ("TO_TIMESTAMP", "select TO_TIMESTAMP(s.datez) from S3Object as s", 271 | b'2017-01-02T03:04:05.006+07:30\n'), 272 | ("UTCNOW", "select UTCNOW() from S3Object", datetime(1, 1, 1)), 273 | 274 | ] 275 | 276 | try: 277 | test_sql_expressions(client, json_testfile, tests, log_output) 278 | except Exception as select_err: 279 | raise select_err 280 | # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) 281 | # pass 282 | 283 | # Test passes 284 | print(log_output.json_report()) 285 | 286 | 287 | def test_sql_functions_string(client, log_output): 288 | 289 | json_testfile = """ 290 | {"id": 1, "name": "John"} 291 | {"id": 2, "name": " \tfoobar\t "} 292 | {"id": 3, "name": "1112211foobar22211122"} 293 | """ 294 | 295 | tests = [ 296 | # CHAR_LENGTH 297 | ("CHAR_LENGTH", "select CHAR_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'), 298 | ("CHARACTER_LENGTH", 299 | "select CHARACTER_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'), 300 | # LOWER 301 | ("LOWER", "select LOWER(s.name) from S3Object as s where s.id= 1", b'john\n'), 302 | # SUBSTRING 303 | ("SUBSTRING_1", "select SUBSTRING(s.name FROM 2) from S3Object as s where s.id = 1", b'ohn\n'), 304 | ("SUBSTRING_2", "select SUBSTRING(s.name FROM 2 FOR 2) from S3Object as s where s.id = 1", b'oh\n'), 305 | ("SUBSTRING_3", "select SUBSTRING(s.name FROM -1 FOR 2) from S3Object as s where s.id = 1", b'\n'), 306 | # TRIM 307 | ("TRIM_1", "select TRIM(s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'), 308 | ("TRIM_2", "select TRIM(LEADING FROM s.name) from S3Object as s where s.id = 2", 309 | b'\tfoobar\t \n'), 310 | ("TRIM_3", "select TRIM(TRAILING FROM s.name) from S3Object as s where s.id = 2", 311 | b' \tfoobar\t\n'), 312 | ("TRIM_4", "select TRIM(BOTH FROM s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'), 313 | ("TRIM_5", "select TRIM(BOTH '12' FROM s.name) from S3Object as s where s.id = 3", b'foobar\n'), 314 | # UPPER 315 | ("UPPER", "select UPPER(s.name) from S3Object as s where s.id= 1", b'JOHN\n'), 316 | ] 317 | 318 | try: 319 | test_sql_expressions(client, json_testfile, tests, log_output) 320 | except Exception as select_err: 321 | raise select_err 322 | # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) 323 | # pass 324 | 325 | # Test passes 326 | print(log_output.json_report()) 327 | 328 | 329 | def test_sql_datatypes(client, log_output): 330 | json_testfile = """ 331 | {"name": "John"} 332 | """ 333 | tests = [ 334 | ("bool", "select CAST('true' AS BOOL) from S3Object", b'true\n'), 335 | ("int", "select CAST('13' AS INT) from S3Object", b'13\n'), 336 | ("integer", "select CAST('13' AS INTEGER) from S3Object", b'13\n'), 337 | ("string", "select CAST(true AS STRING) from S3Object", b'true\n'), 338 | ("float", "select CAST('13.3' AS FLOAT) from S3Object", b'13.3\n'), 339 | ("decimal", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'), 340 | ("numeric", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'), 341 | ("timestamp", "select CAST('2007-04-05T14:30Z' AS TIMESTAMP) from S3Object", 342 | b'2007-04-05T14:30Z\n'), 343 | ] 344 | 345 | try: 346 | test_sql_expressions(client, json_testfile, tests, log_output) 347 | except Exception as select_err: 348 | raise select_err 349 | # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) 350 | # pass 351 | 352 | # Test passes 353 | print(log_output.json_report()) 354 | 355 | 356 | def test_sql_select(client, log_output): 357 | 358 | json_testfile = """{"id": 1, "created": "June 27", "modified": "July 6" } 359 | {"id": 2, "Created": "June 28", "Modified": "July 7", "Cast": "Random Date" }""" 360 | tests = [ 361 | ("select_1", "select * from S3Object", 362 | b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'), 363 | ("select_2", "select * from S3Object s", 364 | b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'), 365 | ("select_3", "select * from S3Object as s", 366 | b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'), 367 | ("select_4", "select s.line from S3Object as s", b'\n\n'), 368 | ("select_5", 'select s."Created" from S3Object as s', b'\nJune 28\n'), 369 | ("select_5", 'select s."Cast" from S3Object as s', b'\nRandom Date\n'), 370 | ("where", 'select s.created from S3Object as s', b'June 27\nJune 28\n'), 371 | ("limit", 'select * from S3Object as s LIMIT 1', b'1,June 27,July 6\n'), 372 | ] 373 | 374 | try: 375 | test_sql_expressions(client, json_testfile, tests, log_output) 376 | except Exception as select_err: 377 | raise select_err 378 | # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) 379 | # pass 380 | 381 | # Test passes 382 | print(log_output.json_report()) 383 | 384 | 385 | def test_sql_select_json(client, log_output): 386 | json_testcontent = """{ "Rules": [ {"id": "1"}, {"expr": "y > x"}, {"id": "2", "expr": "z = DEBUG"} ]} 387 | { "created": "June 27", "modified": "July 6" } 388 | """ 389 | tests = [ 390 | ("select_1", "SELECT id FROM S3Object[*].Rules[*].id", 391 | b'{"id":"1"}\n{}\n{"id":"2"}\n{}\n'), 392 | ("select_2", 393 | "SELECT id FROM S3Object[*].Rules[*].id WHERE id IS NOT MISSING", b'{"id":"1"}\n{"id":"2"}\n'), 394 | ("select_3", "SELECT d.created, d.modified FROM S3Object[*] d", 395 | b'{}\n{"created":"June 27","modified":"July 6"}\n'), 396 | ("select_4", "SELECT _1.created, _1.modified FROM S3Object[*]", 397 | b'{}\n{"created":"June 27","modified":"July 6"}\n'), 398 | ("select_5", 399 | "Select s.rules[1].expr from S3Object s", b'{"expr":"y > x"}\n{}\n'), 400 | ] 401 | 402 | input_serialization = JSONInputSerialization(json_type=JSON_TYPE_DOCUMENT) 403 | output_serialization = JSONOutputSerialization() 404 | try: 405 | test_sql_expressions_custom_input_output(client, json_testcontent, 406 | input_serialization, output_serialization, tests, log_output) 407 | except Exception as select_err: 408 | raise select_err 409 | # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) 410 | # pass 411 | 412 | # Test passes 413 | print(log_output.json_report()) 414 | 415 | 416 | def test_sql_select_csv_no_header(client, log_output): 417 | json_testcontent = """val1,val2,val3 418 | val4,val5,val6 419 | """ 420 | tests = [ 421 | ("select_1", "SELECT s._2 FROM S3Object as s", b'val2\nval5\n'), 422 | ] 423 | 424 | input_serialization = CSVInputSerialization( 425 | file_header_info=FILE_HEADER_INFO_NONE, 426 | allow_quoted_record_delimiter="FALSE", 427 | ) 428 | output_serialization = CSVOutputSerialization() 429 | try: 430 | test_sql_expressions_custom_input_output(client, json_testcontent, 431 | input_serialization, output_serialization, tests, log_output) 432 | except Exception as select_err: 433 | raise select_err 434 | # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) 435 | # pass 436 | 437 | # Test passes 438 | print(log_output.json_report()) 439 | -------------------------------------------------------------------------------- /run/core/aws-sdk-ruby/aws-stub-tests.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # 3 | # Mint (C) 2017 Minio, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | require 'aws-sdk' 19 | require 'securerandom' 20 | require 'net/http' 21 | require 'multipart_body' 22 | 23 | # For aws-sdk ruby tests to run, setting the following 24 | # environment variables is mandatory. 25 | # SERVER_ENDPOINT: address of the minio server tests will run against 26 | # ACCESS_KEY: access key for the minio server 27 | # SECRET_KEY: secreet key for the minio server 28 | # SERVER_REGION: region minio server is setup to run 29 | # ENABLE_HTTPS: (1|0) turn on/off to specify https or 30 | # http services minio server is running on 31 | # MINT_DATA_DIR: Data directory where test data files are stored 32 | 33 | class AwsSdkRubyTest 34 | # Set variables necessary to create an s3 client instance. 35 | # Get them from the environment variables 36 | 37 | # Region information, eg. "us-east-1" 38 | region = ENV['SERVER_REGION'] ||= 'SERVER_REGION is not set' 39 | # Minio server, eg. "play.minio.io:9000" 40 | access_key_id = ENV['ACCESS_KEY'] ||= 'ACCESS_KEY is not set' 41 | secret_access_key = ENV['SECRET_KEY'] ||= 'SECRET_KEY is not set' 42 | enable_https = ENV['ENABLE_HTTPS'] 43 | end_point = ENV['SERVER_ENDPOINT'] ||= 'SERVER_ENDPOINT is not set' 44 | endpoint = enable_https == '1' ? 'https://' + end_point : 'http://' + end_point 45 | 46 | # Create s3 resource instance,"s3" 47 | @@s3 = Aws::S3::Resource.new( 48 | region: region, 49 | endpoint: endpoint, 50 | access_key_id: access_key_id, 51 | secret_access_key: secret_access_key, 52 | force_path_style: true) 53 | 54 | def initialize_log_output(meth, alert = nil) 55 | # Initialize and return log content in log_output hash table 56 | 57 | # Collect args in args_arr 58 | args_arr = method(meth).parameters.flatten.map(&:to_s) 59 | .reject { |x| x == 'req' || x == 'opt' } 60 | # Create and return log output content 61 | { name: 'aws-sdk-ruby', 62 | function: "#{meth}(#{args_arr.join(',')})", # method name and arguments 63 | args: args_arr, # array of arg names. This'll be replaced with a 64 | # a arg/value pairs insdie the caller method 65 | duration: 0, # test runtime duration in seconds 66 | alert: alert, 67 | message: nil, 68 | error: nil } 69 | end 70 | 71 | def random_bucket_name 72 | 'aws-sdk-ruby-bucket-' + SecureRandom.hex(6) 73 | end 74 | 75 | def calculate_duration(t2, t1) 76 | # Durations are in miliseconds, with precision of 2 decimal places 77 | ((t2 - t1) * 1000).round(2) 78 | end 79 | 80 | def print_log(log_output, start_time) 81 | # Calculate duration in miliseconds 82 | log_output[:duration] = calculate_duration(Time.now, start_time) 83 | # Get rid of the log_output fields if nil 84 | puts log_output.delete_if{|k, value| value == nil}.to_json 85 | # Exit at the first failure 86 | exit 1 if log_output[:status] == 'FAIL' 87 | end 88 | 89 | def cleanUp(buckets, log_output) 90 | # Removes objects and bucket if bucket exists 91 | bucket_name = '' 92 | buckets.each do |b| 93 | bucket_name = b 94 | if bucketExistsWrapper(b, log_output) 95 | removeObjectsWrapper(b, log_output) 96 | removeBucketWrapper(b, log_output) 97 | end 98 | end 99 | rescue => e 100 | raise "Failed to clean-up bucket '#{bucket_name}', #{e}" 101 | end 102 | 103 | # 104 | # API commands/methods 105 | # 106 | def makeBucket(bucket_name) 107 | # Creates a bucket, "bucket_name" 108 | # on S3 client , "s3". 109 | # Returns bucket_name if already exists 110 | @@s3.bucket(bucket_name).exists? ? @@s3.bucket(bucket_name) : @@s3.create_bucket(bucket: bucket_name) 111 | rescue => e 112 | raise e 113 | end 114 | 115 | def makeBucketWrapper(bucket_name, log_output) 116 | makeBucket(bucket_name) 117 | rescue => e 118 | log_output[:function] = "makeBucket(bucket_name)" 119 | log_output[:args] = {'bucket_name': bucket_name} 120 | raise e 121 | end 122 | 123 | def removeBucket(bucket_name) 124 | # Deletes/removes bucket, "bucket_name" on S3 client, "s3" 125 | @@s3.bucket(bucket_name).delete 126 | rescue => e 127 | raise e 128 | end 129 | 130 | def removeBucketWrapper(bucket_name, log_output) 131 | removeBucket(bucket_name) 132 | rescue => e 133 | log_output[:function] = "removeBucket(bucket_name)" 134 | log_output[:args] = {'bucket_name': bucket_name} 135 | raise e 136 | end 137 | 138 | def putObject(bucket_name, file) 139 | # Creates "file" (full path) in bucket, "bucket_name", 140 | # on S3 client, "s3" 141 | file_name = File.basename(file) 142 | @@s3.bucket(bucket_name).object(file_name).upload_file(file) 143 | rescue => e 144 | raise e 145 | end 146 | 147 | def putObjectWrapper(bucket_name, file, log_output) 148 | putObject(bucket_name, file) 149 | rescue => e 150 | log_output[:function] = "putObject(bucket_name, file)" 151 | log_output[:args] = {'bucket_name': bucket_name, 152 | 'file': file} 153 | raise e 154 | end 155 | 156 | def getObject(bucket_name, file, destination) 157 | # Gets/Downloads file, "file", 158 | # from bucket, "bucket_name", of S3 client, "s3" 159 | file_name = File.basename(file) 160 | dest = File.join(destination, file_name) 161 | @@s3.bucket(bucket_name).object(file_name).get(response_target: dest) 162 | rescue => e 163 | raise e 164 | end 165 | 166 | def getObjectWrapper(bucket_name, file, destination, log_output) 167 | getObject(bucket_name, file, destination) 168 | rescue => e 169 | log_output[:function] = "getObject(bucket_name, file)" 170 | log_output[:args] = {'bucket_name': bucket_name, 171 | 'file': file, 172 | 'destination': destination} 173 | raise e 174 | end 175 | 176 | def copyObject(source_bucket_name, target_bucket_name, source_file_name, target_file_name = '') 177 | # Copies file, "file_name", from source bucket, 178 | # "source_bucket_name", to target bucket, 179 | # "target_bucket_name", on S3 client, "s3" 180 | target_file_name = source_file_name if target_file_name.empty? 181 | source = @@s3.bucket(source_bucket_name) 182 | target = @@s3.bucket(target_bucket_name) 183 | source_obj = source.object(source_file_name) 184 | target_obj = target.object(target_file_name) 185 | source_obj.copy_to(target_obj) 186 | rescue => e 187 | raise e 188 | end 189 | 190 | def copyObjectWrapper(source_bucket_name, target_bucket_name, source_file_name, target_file_name = '', log_output) 191 | copyObject(source_bucket_name, target_bucket_name, source_file_name, target_file_name) 192 | rescue => e 193 | log_output[:function] = 'copyObject(source_bucket_name, target_bucket_name, source_file_name, target_file_name = '')' 194 | log_output[:args] = {'source_bucket_name': source_bucket_name, 195 | 'target_bucket_name': target_bucket_name, 196 | 'source_file_name': source_file_name, 197 | 'target_file_name': target_file_name} 198 | raise e 199 | end 200 | 201 | def removeObject(bucket_name, file) 202 | # Deletes file in bucket, 203 | # "bucket_name", on S3 client, "s3". 204 | # If file, "file_name" does not exist, 205 | # it quietly returns without any error message 206 | @@s3.bucket(bucket_name).object(file).delete 207 | rescue => e 208 | raise e 209 | end 210 | 211 | def removeObjectWrapper(bucket_name, file_name, log_output) 212 | removeObject(bucket_name, file_name) 213 | rescue => e 214 | log_output[:function] = "removeObject(bucket_name, file_name)" 215 | log_output[:args] = {'bucket_name': bucket_name, 216 | 'file_name': file_name} 217 | raise e 218 | end 219 | 220 | def removeObjects(bucket_name) 221 | # Deletes all files in bucket, "bucket_name" 222 | # on S3 client, "s3" 223 | file_name = '' 224 | @@s3.bucket(bucket_name).objects.each do |obj| 225 | file_name = obj.key 226 | obj.delete 227 | end 228 | rescue => e 229 | raise "File name: '#{file_name}', #{e}" 230 | end 231 | 232 | def removeObjectsWrapper(bucket_name, log_output) 233 | removeObjects(bucket_name) 234 | rescue => e 235 | log_output[:function] = 'removeObjects(bucket_name)' 236 | log_output[:args] = {'bucket_name': bucket_name} 237 | raise e 238 | end 239 | 240 | def listBuckets 241 | # Returns an array of bucket names on S3 client, "s3" 242 | bucket_name_list = [] 243 | @@s3.buckets.each do |b| 244 | bucket_name_list.push(b.name) 245 | end 246 | return bucket_name_list 247 | rescue => e 248 | raise e 249 | end 250 | 251 | def listBucketsWrapper(log_output) 252 | listBuckets 253 | rescue => e 254 | log_output[:function] = 'listBuckets' 255 | log_output[:args] = {} 256 | raise e 257 | end 258 | 259 | def listObjects(bucket_name) 260 | # Returns an array of object/file names 261 | # in bucket, "bucket_name", on S3 client, "s3" 262 | object_list = [] 263 | @@s3.bucket(bucket_name).objects.each do |obj| 264 | object_list.push(obj.key) 265 | end 266 | return object_list 267 | rescue => e 268 | raise e 269 | end 270 | 271 | def listObjectsWrapper(bucket_name, log_output) 272 | listObjects(bucket_name) 273 | rescue => e 274 | log_output[:function] = 'listObjects(bucket_name)' 275 | log_output[:args] = {'bucket_name': bucket_name} 276 | raise e 277 | end 278 | 279 | def statObject(bucket_name, file_name) 280 | return @@s3.bucket(bucket_name).object(file_name).exists? 281 | rescue => e 282 | raise e 283 | end 284 | 285 | def statObjectWrapper(bucket_name, file_name, log_output) 286 | statObject(bucket_name, file_name) 287 | rescue => e 288 | log_output[:function] = 'statObject(bucket_name, file_name)' 289 | log_output[:args] = {'bucket_name': bucket_name, 290 | 'file_name': file_name} 291 | raise e 292 | end 293 | 294 | def bucketExists?(bucket_name) 295 | # Returns true if bucket, "bucket_name", exists, 296 | # false otherwise 297 | return @@s3.bucket(bucket_name).exists? 298 | rescue => e 299 | raise e 300 | end 301 | 302 | def bucketExistsWrapper(bucket_name, log_output) 303 | bucketExists?(bucket_name) 304 | rescue => e 305 | log_output[:function] = 'bucketExists?(bucket_name)' 306 | log_output[:args] = {'bucket_name': bucket_name} 307 | raise e 308 | end 309 | 310 | def presignedGet(bucket_name, file_name) 311 | # Returns download/get url 312 | obj = @@s3.bucket(bucket_name).object(file_name) 313 | return obj.presigned_url(:get, expires_in: 600) 314 | rescue => e 315 | raise e 316 | end 317 | 318 | def presignedGetWrapper(bucket_name, file_name, log_output) 319 | presignedGet(bucket_name, file_name) 320 | rescue => e 321 | log_output[:function] = 'presignedGet(bucket_name, file_name)' 322 | log_output[:args] = {'bucket_name': bucket_name, 323 | 'file_name': file_name} 324 | raise e 325 | end 326 | 327 | def presignedPut(bucket_name, file_name) 328 | # Returns put url 329 | obj = @@s3.bucket(bucket_name).object(file_name) 330 | return obj.presigned_url(:put, expires_in: 600) 331 | rescue => e 332 | raise e 333 | end 334 | 335 | def presignedPutWrapper(bucket_name, file_name, log_output) 336 | presignedPut(bucket_name, file_name) 337 | rescue => e 338 | log_output[:function] = 'presignedPut(bucket_name, file_name)' 339 | log_output[:args] = {'bucket_name': bucket_name, 340 | 'file_name': file_name} 341 | raise e 342 | end 343 | 344 | def presignedPost(bucket_name, file_name, expires_in_sec, max_byte_size) 345 | # Returns upload/post url 346 | obj = @@s3.bucket(bucket_name).object(file_name) 347 | return obj.presigned_post(expires: Time.now + expires_in_sec, 348 | content_length_range: 1..max_byte_size) 349 | rescue => e 350 | raise e 351 | end 352 | 353 | def presignedPostWrapper(bucket_name, file_name, expires_in_sec, max_byte_size, log_output) 354 | presignedPost(bucket_name, file_name, expires_in_sec, max_byte_size) 355 | rescue => e 356 | log_output[:function] = 'presignedPost(bucket_name, file_name, expires_in_sec, max_byte_size)' 357 | log_output[:args] = {'bucket_name': bucket_name, 358 | 'file_name': file_name, 359 | 'expires_in_sec': expires_in_sec, 360 | 'max_byte_size': max_byte_size} 361 | raise e 362 | end 363 | 364 | # To be addressed. S3 API 'get_bucket_policy' does not work! 365 | # def getBucketPolicy(bucket_name) 366 | # # Returns bucket policy 367 | # return @@s3.bucket(bucket_name).get_bucket_policy 368 | # rescue => e 369 | # raise e 370 | # end 371 | 372 | # 373 | # Test case methods 374 | # 375 | def listBucketsTest 376 | # Tests listBuckets api command by creating 377 | # new buckets from bucket_name_list 378 | 379 | # get 2 different random bucket names and create a list 380 | bucket_name_list = [random_bucket_name, random_bucket_name] 381 | # Initialize hash table, 'log_output' 382 | log_output = initialize_log_output('listBuckets') 383 | # Prepare arg/value hash table and set it in log_output 384 | arg_value_hash = {} 385 | log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } 386 | log_output[:args] = arg_value_hash 387 | 388 | begin 389 | start_time = Time.now 390 | prev_total_buckets = listBucketsWrapper(log_output).length 391 | new_buckets = bucket_name_list.length 392 | bucket_name_list.each do |b| 393 | makeBucketWrapper(b, log_output) 394 | end 395 | new_total_buckets = prev_total_buckets + new_buckets 396 | if new_total_buckets >= prev_total_buckets + new_buckets 397 | log_output[:status] = 'PASS' 398 | else 399 | log_output[:error] = 'Could not find expected number of buckets' 400 | log_output[:status] = 'FAIL' 401 | end 402 | cleanUp(bucket_name_list, log_output) 403 | rescue => log_output[:error] 404 | log_output[:status] = 'FAIL' 405 | end 406 | 407 | print_log(log_output, start_time) 408 | end 409 | 410 | def makeBucketTest 411 | # Tests makeBucket api command. 412 | 413 | # get random bucket name 414 | bucket_name = random_bucket_name 415 | # Initialize hash table, 'log_output' 416 | log_output = initialize_log_output('makeBucket') 417 | # Prepare arg/value hash table and set it in log_output 418 | arg_value_hash = {} 419 | log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } 420 | log_output[:args] = arg_value_hash 421 | 422 | begin 423 | start_time = Time.now 424 | makeBucketWrapper(bucket_name, log_output) 425 | 426 | if bucketExistsWrapper(bucket_name, log_output) 427 | log_output[:status] = 'PASS' 428 | else 429 | log_output[:error] = 'Bucket expected to be created does not exist' 430 | log_output[:status] = 'FAIL' 431 | end 432 | cleanUp([bucket_name], log_output) 433 | rescue => log_output[:error] 434 | log_output[:status] = 'FAIL' 435 | end 436 | 437 | print_log(log_output, start_time) 438 | end 439 | 440 | def bucketExistsNegativeTest 441 | # Tests bucketExists api command. 442 | 443 | # get random bucket name 444 | bucket_name = random_bucket_name 445 | # Initialize hash table, 'log_output' 446 | log_output = initialize_log_output('bucketExists?') 447 | # Prepare arg/value hash table and set it in log_output 448 | arg_value_hash = {} 449 | log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } 450 | log_output[:args] = arg_value_hash 451 | 452 | begin 453 | start_time = Time.now 454 | if !bucketExistsWrapper(bucket_name, log_output) 455 | log_output[:status] = 'PASS' 456 | else 457 | log_output[:error] = "Failed to return 'false' for a non-existing bucket" 458 | log_output[:status] = 'FAIL' 459 | end 460 | cleanUp([bucket_name], log_output) 461 | rescue => log_output[:error] 462 | log_output[:status] = 'FAIL' 463 | end 464 | 465 | print_log(log_output, start_time) 466 | end 467 | 468 | def removeBucketTest 469 | # Tests removeBucket api command. 470 | 471 | # get a random bucket name 472 | bucket_name = random_bucket_name 473 | # Initialize hash table, 'log_output' 474 | log_output = initialize_log_output('removeBucket') 475 | # Prepare arg/value hash table and set it in log_output 476 | arg_value_hash = {} 477 | log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } 478 | log_output[:args] = arg_value_hash 479 | 480 | begin 481 | start_time = Time.now 482 | makeBucketWrapper(bucket_name, log_output) 483 | removeBucketWrapper(bucket_name, log_output) 484 | if !bucketExistsWrapper(bucket_name, log_output) 485 | log_output[:status] = 'PASS' 486 | else 487 | log_output[:error] = 'Bucket expected to be removed still exists' 488 | log_output[:status] = 'FAIL' 489 | end 490 | cleanUp([bucket_name], log_output) 491 | rescue => log_output[:error] 492 | log_output[:status] = 'FAIL' 493 | end 494 | 495 | print_log(log_output, start_time) 496 | end 497 | 498 | def putObjectTest(file) 499 | # Tests putObject api command by uploading a file 500 | 501 | # get random bucket name 502 | bucket_name = random_bucket_name 503 | # Initialize hash table, 'log_output' 504 | log_output = initialize_log_output('putObject') 505 | # Prepare arg/value hash table and set it in log_output 506 | arg_value_hash = {} 507 | log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } 508 | log_output[:args] = arg_value_hash 509 | 510 | begin 511 | start_time = Time.now 512 | makeBucketWrapper(bucket_name, log_output) 513 | putObjectWrapper(bucket_name, file, log_output) 514 | if statObjectWrapper(bucket_name, File.basename(file), log_output) 515 | log_output[:status] = 'PASS' 516 | else 517 | log_output[:error] = "Status for the created object returned 'false'" 518 | log_output[:status] = 'FAIL' 519 | end 520 | cleanUp([bucket_name], log_output) 521 | rescue => log_output[:error] 522 | log_output[:status] = 'FAIL' 523 | end 524 | 525 | print_log(log_output, start_time) 526 | end 527 | 528 | def removeObjectTest(file) 529 | # Tests removeObject api command by uploading and removing a file 530 | 531 | # get random bucket name 532 | bucket_name = random_bucket_name 533 | # Initialize hash table, 'log_output' 534 | log_output = initialize_log_output('removeObject') 535 | # Prepare arg/value hash table and set it in log_output 536 | arg_value_hash = {} 537 | log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } 538 | log_output[:args] = arg_value_hash 539 | 540 | begin 541 | start_time = Time.now 542 | makeBucketWrapper(bucket_name, log_output) 543 | putObjectWrapper(bucket_name, file, log_output) 544 | removeObjectWrapper(bucket_name, File.basename(file), log_output) 545 | if !statObjectWrapper(bucket_name, File.basename(file), log_output) 546 | log_output[:status] = 'PASS' 547 | else 548 | log_output[:error] = "Status for the removed object returned 'true'" 549 | log_output[:status] = 'FAIL' 550 | end 551 | cleanUp([bucket_name], log_output) 552 | rescue => log_output[:error] 553 | log_output[:status] = 'FAIL' 554 | end 555 | 556 | print_log(log_output, start_time) 557 | end 558 | 559 | def getObjectTest(file, destination) 560 | # Tests getObject api command 561 | 562 | # get random bucket name 563 | bucket_name = random_bucket_name 564 | # Initialize hash table, 'log_output' 565 | log_output = initialize_log_output('getObject') 566 | # Prepare arg/value hash table and set it in log_output 567 | arg_value_hash = {} 568 | log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } 569 | log_output[:args] = arg_value_hash 570 | 571 | begin 572 | start_time = Time.now 573 | makeBucketWrapper(bucket_name, log_output) 574 | putObjectWrapper(bucket_name, file, log_output) 575 | getObjectWrapper(bucket_name, file, destination, log_output) 576 | if system("ls -l #{destination} > /dev/null") 577 | log_output[:status] = 'PASS' 578 | else 579 | log_output[:error] = "Downloaded object does not exist at #{destination}" 580 | log_output[:status] = 'FAIL' 581 | end 582 | cleanUp([bucket_name], log_output) 583 | rescue => log_output[:error] 584 | log_output[:status] = 'FAIL' 585 | end 586 | 587 | print_log(log_output, start_time) 588 | end 589 | 590 | def listObjectsTest(file_list) 591 | # Tests listObjects api command 592 | 593 | # get random bucket name 594 | bucket_name = random_bucket_name 595 | # Initialize hash table, 'log_output' 596 | log_output = initialize_log_output('listObjects') 597 | # Prepare arg/value hash table and set it in log_output 598 | arg_value_hash = {} 599 | log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } 600 | log_output[:args] = arg_value_hash 601 | 602 | begin 603 | start_time = Time.now 604 | makeBucketWrapper(bucket_name, log_output) 605 | # Put all objects into the bucket 606 | file_list.each do |f| 607 | putObjectWrapper(bucket_name, f, log_output) 608 | end 609 | # Total number of files uploaded 610 | expected_no = file_list.length 611 | # Actual number is what api returns 612 | actual_no = listObjectsWrapper(bucket_name, log_output).length 613 | # Compare expected and actual values 614 | if expected_no == actual_no 615 | log_output[:status] = 'PASS' 616 | else 617 | log_output[:error] = 'Expected and actual number of listed files/objects do not match!' 618 | log_output[:status] = 'FAIL' 619 | end 620 | cleanUp([bucket_name], log_output) 621 | rescue => log_output[:error] 622 | log_output[:status] = 'FAIL' 623 | end 624 | 625 | print_log(log_output, start_time) 626 | end 627 | 628 | def copyObjectTest(data_dir, source_file_name, target_file_name = '') 629 | # Tests copyObject api command 630 | 631 | # get random bucket names 632 | source_bucket_name = random_bucket_name 633 | target_bucket_name = random_bucket_name 634 | # Initialize hash table, 'log_output' 635 | log_output = initialize_log_output('copyObject') 636 | # Prepare arg/value hash table and set it in log_output 637 | arg_value_hash = {} 638 | log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } 639 | log_output[:args] = arg_value_hash 640 | 641 | begin 642 | start_time = Time.now 643 | target_file_name = source_file_name if target_file_name.empty? 644 | makeBucketWrapper(source_bucket_name, log_output) 645 | makeBucketWrapper(target_bucket_name, log_output) 646 | putObjectWrapper(source_bucket_name, 647 | File.join(data_dir, source_file_name), log_output) 648 | copyObjectWrapper(source_bucket_name, target_bucket_name, 649 | source_file_name, target_file_name, log_output) 650 | # Check if copy worked fine 651 | if statObjectWrapper(target_bucket_name, target_file_name, log_output) 652 | log_output[:status] = 'PASS' 653 | else 654 | log_output[:error] = 'Copied file could not be found in the expected location' 655 | log_output[:status] = 'FAIL' 656 | end 657 | cleanUp([source_bucket_name, target_bucket_name], log_output) 658 | rescue => log_output[:error] 659 | log_output[:status] = 'FAIL' 660 | end 661 | 662 | print_log(log_output, start_time) 663 | end 664 | 665 | def presignedGetObjectTest(data_dir, file_name) 666 | # Tests presignedGetObject api command 667 | 668 | # get random bucket name 669 | bucket_name = random_bucket_name 670 | # Initialize hash table, 'log_output' 671 | log_output = initialize_log_output('presignedGet') 672 | # Prepare arg/value hash table and set it in log_output 673 | arg_value_hash = {} 674 | log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } 675 | log_output[:args] = arg_value_hash 676 | 677 | begin 678 | start_time = Time.now 679 | makeBucketWrapper(bucket_name, log_output) 680 | file = File.join(data_dir, file_name) 681 | # Get check sum value without the file name 682 | cksum_orig = `cksum #{file}`.split[0..1] 683 | putObjectWrapper(bucket_name, file, log_output) 684 | get_url = presignedGetWrapper(bucket_name, file_name, log_output) 685 | # Download the file using the URL 686 | # generated by presignedGet api command 687 | `wget -O /tmp/#{file_name} '#{get_url}' > /dev/null 2>&1` 688 | # Get check sum value for the downloaded file 689 | # Split to get rid of the file name 690 | cksum_new = `cksum /tmp/#{file_name}`.split[0..1] 691 | 692 | # Check if check sum values for the orig file 693 | # and the downloaded file match 694 | if cksum_orig == cksum_new 695 | log_output[:status] = 'PASS' 696 | else 697 | log_output[:error] = 'Check sum values do NOT match' 698 | log_output[:status] = 'FAIL' 699 | end 700 | cleanUp([bucket_name], log_output) 701 | rescue => log_output[:error] 702 | log_output[:status] = 'FAIL' 703 | end 704 | 705 | print_log(log_output, start_time) 706 | end 707 | 708 | def presignedPutObjectTest(data_dir, file_name) 709 | # Tests presignedPutObject api command 710 | 711 | # get random bucket name 712 | bucket_name = random_bucket_name 713 | # Initialize hash table, 'log_output' 714 | log_output = initialize_log_output('presignedPut') 715 | # Prepare arg/value hash table and set it in log_output 716 | arg_value_hash = {} 717 | log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } 718 | log_output[:args] = arg_value_hash 719 | 720 | begin 721 | start_time = Time.now 722 | makeBucketWrapper(bucket_name, log_output) 723 | file = File.join(data_dir, file_name) 724 | 725 | # Get check sum value and 726 | # split to get rid of the file name 727 | cksum_orig = `cksum #{file}`.split[0..1] 728 | 729 | # Generate presigned Put URL and parse it 730 | uri = URI.parse(presignedPutWrapper(bucket_name, file_name, log_output)) 731 | request = Net::HTTP::Put.new(uri.request_uri, 'content-type' => 'application/octet-stream', 732 | 'x-amz-acl' => 'public-read') 733 | request.body = IO.read(File.join(data_dir, file_name)) 734 | 735 | http = Net::HTTP.new(uri.host, uri.port) 736 | http.use_ssl = true if ENV['ENABLE_HTTPS'] == '1' 737 | 738 | http.request(request) 739 | 740 | if statObjectWrapper(bucket_name, file_name, log_output) 741 | getObjectWrapper(bucket_name, file_name, '/tmp', log_output) 742 | cksum_new = `cksum /tmp/#{file_name}`.split[0..1] 743 | # Check if check sum values of the orig file 744 | # and the downloaded file match 745 | if cksum_orig == cksum_new 746 | log_output[:status] = 'PASS' 747 | else 748 | log_output[:error] = 'Check sum values do NOT match' 749 | log_output[:status] = 'FAIL' 750 | end 751 | else 752 | log_output[:error] = 'Expected to be created object does NOT exist' 753 | log_output[:status] = 'FAIL' 754 | end 755 | cleanUp([bucket_name], log_output) 756 | rescue => log_output[:error] 757 | log_output[:status] = 'FAIL' 758 | end 759 | 760 | print_log(log_output, start_time) 761 | end 762 | 763 | def presignedPostObjectTest(data_dir, file_name, 764 | expires_in_sec, max_byte_size) 765 | # Tests presignedPostObject api command 766 | 767 | # get random bucket name 768 | bucket_name = random_bucket_name 769 | # Initialize hash table, 'log_output' 770 | log_output = initialize_log_output('presignedPost') 771 | # Prepare arg/value hash table and set it in log_output 772 | arg_value_hash = {} 773 | log_output[:args].each { |x| arg_value_hash[:"#{x}"] = eval x.to_s } 774 | log_output[:args] = arg_value_hash 775 | 776 | begin 777 | start_time = Time.now 778 | makeBucketWrapper(bucket_name, log_output) 779 | file = File.join(data_dir, file_name) 780 | 781 | # Get check sum value and split it 782 | # into parts to get rid of the file name 783 | cksum_orig = `cksum #{file}`.split[0..1] 784 | # Create the presigned POST url 785 | post = presignedPostWrapper(bucket_name, file_name, 786 | expires_in_sec, max_byte_size, log_output) 787 | 788 | # Prepare multi parts array for POST command request 789 | file_part = Part.new name: 'file', 790 | body: IO.read(File.join(data_dir, file_name)), 791 | filename: file_name, 792 | content_type: 'application/octet-stream' 793 | parts = [] 794 | # Add POST fields into parts array 795 | post.fields.each do |field, value| 796 | parts.push(Part.new(field, value)) 797 | end 798 | parts.push(file_part) 799 | boundary = "---------------------------#{rand(10_000_000_000_000_000)}" 800 | body_parts = MultipartBody.new parts, boundary 801 | 802 | # Parse presigned Post URL 803 | uri = URI.parse(post.url) 804 | 805 | # Create the HTTP objects 806 | http = Net::HTTP.new(uri.host, uri.port) 807 | http.use_ssl = true if ENV['ENABLE_HTTPS'] == '1' 808 | request = Net::HTTP::Post.new(uri.request_uri) 809 | request.body = body_parts.to_s 810 | request.content_type = "multipart/form-data; boundary=#{boundary}" 811 | # Send the request 812 | log_output[:error] = http.request(request) 813 | 814 | if statObjectWrapper(bucket_name, file_name, log_output) 815 | getObjectWrapper(bucket_name, file_name, '/tmp', log_output) 816 | cksum_new = `cksum /tmp/#{file_name}`.split[0..1] 817 | # Check if check sum values of the orig file 818 | # and the downloaded file match 819 | if cksum_orig == cksum_new 820 | log_output[:status] = 'PASS' 821 | # FIXME: HTTP No Content error, status code=204 is returned as error 822 | log_output[:error] = nil 823 | else 824 | log_output[:error] = 'Check sum values do NOT match' 825 | log_output[:status] = 'FAIL' 826 | end 827 | else 828 | log_output[:error] = 'Expected to be created object does NOT exist' 829 | log_output[:status] = 'FAIL' 830 | end 831 | cleanUp([bucket_name], log_output) 832 | rescue => log_output[:error] 833 | log_output[:status] = 'FAIL' 834 | end 835 | 836 | print_log(log_output, start_time) 837 | end 838 | end 839 | 840 | # MAIN CODE 841 | 842 | # Create test Class instance and call the tests 843 | aws = AwsSdkRubyTest.new 844 | file_name1 = 'datafile-1-kB' 845 | file_new_name = 'datafile-1-kB-copy' 846 | file_name_list = ['datafile-1-kB', 'datafile-1-b', 'datafile-6-MB'] 847 | # Add data_dir in front of each file name in file_name_list 848 | # The location where the bucket and file 849 | # objects are going to be created. 850 | data_dir = ENV['MINT_DATA_DIR'] ||= 'MINT_DATA_DIR is not set' 851 | file_list = file_name_list.map { |f| File.join(data_dir, f) } 852 | destination = '/tmp' 853 | 854 | aws.listBucketsTest 855 | aws.listObjectsTest(file_list) 856 | aws.makeBucketTest 857 | aws.bucketExistsNegativeTest 858 | aws.removeBucketTest 859 | aws.putObjectTest(File.join(data_dir, file_name1)) 860 | aws.removeObjectTest(File.join(data_dir, file_name1)) 861 | aws.getObjectTest(File.join(data_dir, file_name1), destination) 862 | aws.copyObjectTest(data_dir, file_name1) 863 | aws.copyObjectTest(data_dir, file_name1, file_new_name) 864 | aws.presignedGetObjectTest(data_dir, file_name1) 865 | aws.presignedPutObjectTest(data_dir, file_name1) 866 | aws.presignedPostObjectTest(data_dir, file_name1, 60, 3*1024*1024) 867 | -------------------------------------------------------------------------------- /run/core/aws-sdk-php/quick-tests.php: -------------------------------------------------------------------------------- 1 | 'val-1']; 38 | 39 | /** 40 | * ClientConfig abstracts configuration details to connect to a 41 | * S3-like service 42 | */ 43 | class ClientConfig { 44 | public $creds; 45 | public $endpoint; 46 | public $region; 47 | 48 | function __construct(string $access_key, string $secret_key, string $host, string $secure, string $region) { 49 | $this->creds = new Aws\Credentials\Credentials($access_key, $secret_key); 50 | 51 | if ($secure == "1") { 52 | $this->endpoint = "https://" . $host; 53 | } else { 54 | $this->endpoint = "http://" . $host; 55 | } 56 | 57 | $this->region = $region; 58 | } 59 | } 60 | 61 | /** 62 | * randomName returns a name prefixed by aws-sdk-php using uniqid() 63 | * from standard library 64 | * 65 | * @return string 66 | */ 67 | function randomName():string { 68 | return uniqid("aws-sdk-php-"); 69 | } 70 | 71 | /** 72 | * getStatusCode returns HTTP status code of the given result. 73 | * 74 | * @param $result - AWS\S3 result object 75 | * 76 | * @return string - HTTP status code. E.g, "400" for Bad Request. 77 | */ 78 | function getStatusCode($result):string { 79 | return $result->toArray()['@metadata']['statusCode']; 80 | } 81 | 82 | /** 83 | * runExceptionalTests executes a collection of tests that will throw 84 | * a known exception. 85 | * 86 | * @param $s3Client AWS\S3\S3Client object 87 | * 88 | * @param $apiCall Name of the S3Client API method to call 89 | * 90 | * @param $exceptionMatcher Name of Aws\S3\Exception\S3Exception 91 | * method to fetch exception details 92 | * 93 | * @param $exceptionParamMap Associative array of exception names to 94 | * API parameters. E.g, 95 | * $apiCall = 'headBucket' 96 | * $exceptionMatcher = 'getStatusCode' 97 | * $exceptionParamMap = [ 98 | * // Non existent bucket 99 | * '404' => ['Bucket' => $bucket['Name'] . '--'], 100 | * 101 | * // Non existent bucket 102 | * '404' => ['Bucket' => $bucket['Name'] . '-non-existent'], 103 | * ]; 104 | * 105 | * @return string - HTTP status code. E.g, "404" for Non existent bucket. 106 | */ 107 | function runExceptionalTests($s3Client, $apiCall, $exceptionMatcher, $exceptionParamMap) { 108 | foreach($exceptionParamMap as $exn => $params) { 109 | $exceptionCaught = false; 110 | try { 111 | $result = $s3Client->$apiCall($params); 112 | } catch(Aws\S3\Exception\S3Exception $e) { 113 | $exceptionCaught = true; 114 | switch ($e->$exceptionMatcher()) { 115 | case $exn: 116 | // This is expected 117 | continue 2; 118 | default: 119 | throw $e; 120 | } 121 | } 122 | finally { 123 | if (!$exceptionCaught) { 124 | $message = sprintf("Expected %s to fail with %s", $apiCall, $exn); 125 | throw new Exception($message); 126 | } 127 | } 128 | } 129 | } 130 | 131 | /** 132 | * testListBuckets tests ListBuckets S3 API 133 | * 134 | * @param $s3Client AWS\S3\S3Client object 135 | * 136 | * @return void 137 | */ 138 | function testListBuckets(S3Client $s3Client) { 139 | $buckets = $s3Client->listBuckets(); 140 | $debugger = $GLOBALS['debugger']; 141 | foreach ($buckets['Buckets'] as $bucket){ 142 | $debugger->out($bucket['Name'] . "\n"); 143 | } 144 | } 145 | 146 | /** 147 | * testBucketExists tests HEAD Bucket S3 API 148 | * 149 | * @param $s3Client AWS\S3\S3Client object 150 | * 151 | * @return void 152 | */ 153 | function testBucketExists(S3Client $s3Client) { 154 | // List all buckets 155 | $buckets = $s3Client->listBuckets(); 156 | // All HEAD on existing buckets must return success 157 | foreach($buckets['Buckets'] as $bucket) { 158 | $result = $s3Client->headBucket(['Bucket' => $bucket['Name']]); 159 | if (getStatusCode($result) != HTTP_OK) 160 | throw new Exception('headBucket API failed for ' . $bucket['Name']); 161 | } 162 | 163 | // Run failure tests 164 | $params = [ 165 | // Non existent bucket 166 | '404' => ['Bucket' => $bucket['Name'] . '--'], 167 | 168 | // Non existent bucket 169 | '404' => ['Bucket' => $bucket['Name'] . '-non-existent'], 170 | ]; 171 | runExceptionalTests($s3Client, 'headBucket', 'getStatusCode', $params); 172 | } 173 | 174 | 175 | /** 176 | * testHeadObject tests HeadObject S3 API 177 | * 178 | * @param $s3Client AWS\S3\S3Client object 179 | * 180 | * @param $objects Associative array of buckets and objects 181 | * 182 | * @return void 183 | */ 184 | function testHeadObject($s3Client, $objects) { 185 | foreach($objects as $bucket => $object) { 186 | $result = $s3Client->headObject(['Bucket' => $bucket, 'Key' => $object]); 187 | if (getStatusCode($result) != HTTP_OK) 188 | throw new Exception('headObject API failed for ' . 189 | $bucket . '/' . $object); 190 | if (strtolower(json_encode($result['Metadata'])) != strtolower(json_encode(TEST_METADATA))) { 191 | throw new Exception("headObject API Metadata didn't match for " . 192 | $bucket . '/' . $object); 193 | } 194 | } 195 | 196 | // Run failure tests 197 | $params = [ 198 | '404' => ['Bucket' => $bucket, 'Key' => $object . '-non-existent'] 199 | ]; 200 | runExceptionalTests($s3Client, 'headObject', 'getStatusCode', $params); 201 | } 202 | 203 | /** 204 | * testListObjects tests ListObjectsV1 and V2 S3 APIs 205 | * 206 | * @param $s3Client AWS\S3\S3Client object 207 | * 208 | * @param $params associative array containing bucket and object names 209 | * 210 | * @return void 211 | */ 212 | function testListObjects($s3Client, $params) { 213 | $bucket = $params['Bucket']; 214 | $object = $params['Object']; 215 | $debugger = $GLOBALS['debugger']; 216 | try { 217 | for ($i = 0; $i < 5; $i++) { 218 | $copyKey = $object . '-copy-' . strval($i); 219 | $result = $s3Client->copyObject([ 220 | 'Bucket' => $bucket, 221 | 'Key' => $copyKey, 222 | 'CopySource' => $bucket . '/' . $object, 223 | ]); 224 | if (getStatusCode($result) != HTTP_OK) 225 | throw new Exception("copyObject API failed for " . $bucket . '/' . $object); 226 | } 227 | 228 | $paginator = $s3Client->getPaginator('ListObjects', ['Bucket' => $bucket]); 229 | foreach ($paginator->search('Contents[].Key') as $key) { 230 | $debugger->out('key = ' . $key . "\n"); 231 | } 232 | 233 | $paginator = $s3Client->getPaginator('ListObjectsV2', ['Bucket' => $bucket]); 234 | foreach ($paginator->search('Contents[].Key') as $key) { 235 | $debugger->out('key = ' . $key . "\n"); 236 | } 237 | 238 | $prefix = 'obj'; 239 | $result = $s3Client->listObjects(['Bucket' => $bucket, 'Prefix' => $prefix]); 240 | if (getStatusCode($result) != HTTP_OK || $result['Prefix'] != $prefix) 241 | throw new Exception("listObject API failed for " . $bucket . '/' . $object); 242 | 243 | $maxKeys = 1; 244 | $result = $s3Client->listObjects(['Bucket' => $bucket, 'MaxKeys' => $maxKeys]); 245 | if (getStatusCode($result) != HTTP_OK || count($result['Contents']) != $maxKeys) 246 | throw new Exception("listObject API failed for " . $bucket . '/' . $object); 247 | 248 | $params = [ 249 | 'InvalidArgument' => ['Bucket' => $bucket, 'MaxKeys' => -1], 250 | 'NoSuchBucket' => ['Bucket' => $bucket . '-non-existent'] 251 | ]; 252 | runExceptionalTests($s3Client, 'listObjects', 'getAwsErrorCode', $params); 253 | 254 | } finally { 255 | $s3Client->deleteObjects([ 256 | 'Bucket' => $bucket, 257 | 'Delete' => [ 258 | 'Objects' => array_map(function($a, $b) { 259 | return ['Key' => $a . '-copy-' . strval($b)]; 260 | }, array_fill(0, 5, $object), range(0,4)) 261 | ], 262 | ]); 263 | } 264 | } 265 | 266 | /** 267 | * testListMultipartUploads tests ListMultipartUploads, ListParts and 268 | * UploadPartCopy S3 APIs 269 | * 270 | * @param $s3Client AWS\S3\S3Client object 271 | * 272 | * @param $params associative array containing bucket and object names 273 | * 274 | * @return void 275 | */ 276 | function testListMultipartUploads($s3Client, $params) { 277 | $bucket = $params['Bucket']; 278 | $object = $params['Object']; 279 | $debugger = $GLOBALS['debugger']; 280 | 281 | $data_dir = $GLOBALS['MINT_DATA_DIR']; 282 | // Initiate multipart upload 283 | $result = $s3Client->createMultipartUpload([ 284 | 'Bucket' => $bucket, 285 | 'Key' => $object . '-copy', 286 | ]); 287 | if (getStatusCode($result) != HTTP_OK) 288 | throw new Exception('createMultipartupload API failed for ' . 289 | $bucket . '/' . $object); 290 | 291 | // upload 5 parts 292 | $uploadId = $result['UploadId']; 293 | $parts = []; 294 | try { 295 | for ($i = 0; $i < 5; $i++) { 296 | $result = $s3Client->uploadPartCopy([ 297 | 'Bucket' => $bucket, 298 | 'Key' => $object . '-copy', 299 | 'UploadId' => $uploadId, 300 | 'PartNumber' => $i+1, 301 | 'CopySource' => $bucket . '/' . $object, 302 | ]); 303 | if (getStatusCode($result) != HTTP_OK) { 304 | throw new Exception('uploadPart API failed for ' . 305 | $bucket . '/' . $object); 306 | } 307 | array_push($parts, [ 308 | 'ETag' => $result['ETag'], 309 | 'PartNumber' => $i+1, 310 | ]); 311 | } 312 | 313 | // ListMultipartUploads and ListParts may return empty 314 | // responses in the case of minio gateway gcs and minio server 315 | // FS mode. So, the following tests don't make assumptions on 316 | // result response. 317 | $paginator = $s3Client->getPaginator('ListMultipartUploads', 318 | ['Bucket' => $bucket]); 319 | foreach ($paginator->search('Uploads[].{Key: Key, UploadId: UploadId}') as $keyHash) { 320 | $debugger->out('key = ' . $keyHash['Key'] . ' uploadId = ' . $keyHash['UploadId'] . "\n"); 321 | } 322 | 323 | $paginator = $s3Client->getPaginator('ListParts', [ 324 | 'Bucket' => $bucket, 325 | 'Key' => $object . '-copy', 326 | 'UploadId' => $uploadId, 327 | ]); 328 | foreach ($paginator->search('Parts[].{PartNumber: PartNumber, ETag: ETag}') as $partsHash) { 329 | $debugger->out('partNumber = ' . $partsHash['PartNumber'] . ' ETag = ' . $partsHash['ETag'] . "\n"); 330 | } 331 | 332 | }finally { 333 | $s3Client->abortMultipartUpload([ 334 | 'Bucket' => $bucket, 335 | 'Key' => $object . '-copy', 336 | 'UploadId' => $uploadId 337 | ]); 338 | } 339 | } 340 | 341 | /** 342 | * initSetup creates buckets and objects necessary for the functional 343 | * tests to run 344 | * 345 | * @param $s3Client AWS\S3\S3Client object 346 | * 347 | * @param $objects Associative array of buckets and objects 348 | * 349 | * @return void 350 | */ 351 | function initSetup(S3Client $s3Client, $objects) { 352 | $MINT_DATA_DIR = $GLOBALS['MINT_DATA_DIR']; 353 | foreach($objects as $bucket => $object) { 354 | $s3Client->createBucket(['Bucket' => $bucket]); 355 | $stream = NULL; 356 | try { 357 | if (!file_exists($MINT_DATA_DIR . '/' . FILE_1_KB)) 358 | throw new Exception('File not found ' . $MINT_DATA_DIR . '/' . FILE_1_KB); 359 | 360 | $stream = Psr7\Utils::streamFor(fopen($MINT_DATA_DIR . '/' . FILE_1_KB, 'r')); 361 | $result = $s3Client->putObject([ 362 | 'Bucket' => $bucket, 363 | 'Key' => $object, 364 | 'Body' => $stream, 365 | 'Metadata' => TEST_METADATA, 366 | ]); 367 | if (getStatusCode($result) != HTTP_OK) 368 | throw new Exception("putObject API failed for " . $bucket . '/' . $object); 369 | } 370 | 371 | finally { 372 | // close data file 373 | if (!is_null($stream)) 374 | $stream->close(); 375 | } 376 | } 377 | 378 | // Create an empty bucket for bucket policy + delete tests 379 | $result = $s3Client->createBucket(['Bucket' => $GLOBALS['emptyBucket']]); 380 | if (getStatusCode($result) != HTTP_OK) 381 | throw new Exception("createBucket API failed for " . $bucket); 382 | 383 | } 384 | 385 | 386 | /** 387 | * testGetPutObject tests GET/PUT object S3 API 388 | * 389 | * @param $s3Client AWS\S3\S3Client object 390 | * 391 | * @param $params associative array containing bucket and object names 392 | * 393 | * @return void 394 | */ 395 | function testGetPutObject($s3Client, $params) { 396 | $bucket = $params['Bucket']; 397 | $object = $params['Object']; 398 | 399 | // Upload a 10KB file 400 | $MINT_DATA_DIR = $GLOBALS['MINT_DATA_DIR']; 401 | try { 402 | $stream = Psr7\Utils::streamFor(fopen($MINT_DATA_DIR . '/' . FILE_1_KB, 'r')); 403 | $result = $s3Client->putObject([ 404 | 'Bucket' => $bucket, 405 | 'Key' => $object, 406 | 'Body' => $stream, 407 | ]); 408 | } 409 | finally { 410 | $stream->close(); 411 | } 412 | 413 | if (getStatusCode($result) != HTTP_OK) 414 | throw new Exception("putObject API failed for " . $bucket . '/' . $object); 415 | 416 | // Download the same object and verify size 417 | $result = $s3Client->getObject([ 418 | 'Bucket' => $bucket, 419 | 'Key' => $object, 420 | ]); 421 | if (getStatusCode($result) != HTTP_OK) 422 | throw new Exception("getObject API failed for " . $bucket . '/' . $object); 423 | 424 | $body = $result['Body']; 425 | $bodyLen = 0; 426 | while (!$body->eof()) { 427 | $bodyLen += strlen($body->read(4096)); 428 | } 429 | 430 | if ($bodyLen != 1 * 1024) { 431 | throw new Exception("Object downloaded has different content length than uploaded object " 432 | . $bucket . '/' . $object); 433 | } 434 | } 435 | 436 | /** 437 | * testMultipartUploadFailure tests MultipartUpload failures 438 | * 439 | * @param $s3Client AWS\S3\S3Client object 440 | * 441 | * @param $params associative array containing bucket and object names 442 | * 443 | * @return void 444 | */ 445 | function testMultipartUploadFailure($s3Client, $params) { 446 | $bucket = $params['Bucket']; 447 | $object = $params['Object']; 448 | 449 | $MINT_DATA_DIR = $GLOBALS['MINT_DATA_DIR']; 450 | // Initiate multipart upload 451 | $result = $s3Client->createMultipartUpload([ 452 | 'Bucket' => $bucket, 453 | 'Key' => $object, 454 | ]); 455 | if (getStatusCode($result) != HTTP_OK) 456 | throw new Exception('createMultipartupload API failed for ' . 457 | $bucket . '/' . $object); 458 | 459 | // upload 2 parts 460 | $uploadId = $result['UploadId']; 461 | $parts = []; 462 | try { 463 | for ($i = 0; $i < 2; $i++) { 464 | $stream = Psr7\Utils::streamFor(fopen($MINT_DATA_DIR . '/' . FILE_5_MB, 'r')); 465 | $limitedStream = new Psr7\LimitStream($stream, 4 * 1024 * 1024, 0); 466 | $result = $s3Client->uploadPart([ 467 | 'Bucket' => $bucket, 468 | 'Key' => $object, 469 | 'UploadId' => $uploadId, 470 | 'ContentLength' => 4 * 1024 * 1024, 471 | 'Body' => $limitedStream, 472 | 'PartNumber' => $i+1, 473 | ]); 474 | if (getStatusCode($result) != HTTP_OK) { 475 | throw new Exception('uploadPart API failed for ' . 476 | $bucket . '/' . $object); 477 | } 478 | array_push($parts, [ 479 | 'ETag' => $result['ETag'], 480 | 'PartNumber' => $i+1, 481 | ]); 482 | 483 | $limitedStream->close(); 484 | $limitedStream = NULL; 485 | } 486 | } 487 | finally { 488 | if (!is_null($limitedStream)) 489 | $limitedStream->close(); 490 | } 491 | 492 | $params = [ 493 | 'EntityTooSmall' => [ 494 | 'Bucket' => $bucket, 495 | 'Key' => $object, 496 | 'UploadId' => $uploadId, 497 | 'MultipartUpload' => [ 498 | 'Parts' => $parts, 499 | ], 500 | ], 501 | 'NoSuchUpload' => [ 502 | 'Bucket' => $bucket, 503 | 'Key' => $object, 504 | 'UploadId' => 'non-existent', 505 | 'MultipartUpload' => [ 506 | 'Parts' => $parts, 507 | ], 508 | ], 509 | ]; 510 | try { 511 | runExceptionalTests($s3Client, 'completeMultipartUpload', 'getAwsErrorCode', $params); 512 | }finally { 513 | $s3Client->abortMultipartUpload([ 514 | 'Bucket' => $bucket, 515 | 'Key' => $object, 516 | 'UploadId' => $uploadId 517 | ]); 518 | } 519 | } 520 | 521 | /** 522 | * testMultipartUpload tests MultipartUpload S3 APIs 523 | * 524 | * @param $s3Client AWS\S3\S3Client object 525 | * 526 | * @param $params associative array containing bucket and object names 527 | * 528 | * @return void 529 | */ 530 | function testMultipartUpload($s3Client, $params) { 531 | $bucket = $params['Bucket']; 532 | $object = $params['Object']; 533 | 534 | $MINT_DATA_DIR = $GLOBALS['MINT_DATA_DIR']; 535 | // Initiate multipart upload 536 | $result = $s3Client->createMultipartUpload([ 537 | 'Bucket' => $bucket, 538 | 'Key' => $object, 539 | ]); 540 | if (getStatusCode($result) != HTTP_OK) 541 | throw new Exception('createMultipartupload API failed for ' . 542 | $bucket . '/' . $object); 543 | 544 | // upload 2 parts 545 | $uploadId = $result['UploadId']; 546 | $parts = []; 547 | try { 548 | for ($i = 0; $i < 2; $i++) { 549 | $stream = Psr7\Utils::streamFor(fopen($MINT_DATA_DIR . '/' . FILE_5_MB, 'r')); 550 | $result = $s3Client->uploadPart([ 551 | 'Bucket' => $bucket, 552 | 'Key' => $object, 553 | 'UploadId' => $uploadId, 554 | 'ContentLength' => 5 * 1024 * 1024, 555 | 'Body' => $stream, 556 | 'PartNumber' => $i+1, 557 | ]); 558 | if (getStatusCode($result) != HTTP_OK) { 559 | throw new Exception('uploadPart API failed for ' . 560 | $bucket . '/' . $object); 561 | } 562 | array_push($parts, [ 563 | 'ETag' => $result['ETag'], 564 | 'PartNumber' => $i+1, 565 | ]); 566 | 567 | $stream->close(); 568 | $stream = NULL; 569 | } 570 | } 571 | finally { 572 | if (!is_null($stream)) 573 | $stream->close(); 574 | } 575 | 576 | // complete multipart upload 577 | $result = $s3Client->completeMultipartUpload([ 578 | 'Bucket' => $bucket, 579 | 'Key' => $object, 580 | 'UploadId' => $uploadId, 581 | 'MultipartUpload' => [ 582 | 'Parts' => $parts, 583 | ], 584 | ]); 585 | if (getStatusCode($result) != HTTP_OK) { 586 | throw new Exception('completeMultipartupload API failed for ' . 587 | $bucket . '/' . $object); 588 | } 589 | } 590 | 591 | /** 592 | * testAbortMultipartUpload tests aborting of a multipart upload 593 | * 594 | * @param $s3Client AWS\S3\S3Client object 595 | * 596 | * @param $params associative array containing bucket and object names 597 | * 598 | * @return void 599 | */ 600 | function testAbortMultipartUpload($s3Client, $params) { 601 | $bucket = $params['Bucket']; 602 | $object = $params['Object']; 603 | 604 | $MINT_DATA_DIR = $GLOBALS['MINT_DATA_DIR']; 605 | // Initiate multipart upload 606 | $result = $s3Client->createMultipartUpload([ 607 | 'Bucket' => $bucket, 608 | 'Key' => $object, 609 | ]); 610 | if (getStatusCode($result) != HTTP_OK) 611 | throw new Exception('createMultipartupload API failed for ' . 612 | $bucket . '/' . $object); 613 | 614 | // Abort multipart upload 615 | $uploadId = $result['UploadId']; 616 | $result = $s3Client->abortMultipartUpload([ 617 | 'Bucket' => $bucket, 618 | 'Key' => $object, 619 | 'UploadId' => $uploadId, 620 | ]); 621 | if (getStatusCode($result) != HTTP_NOCONTENT) 622 | throw new Exception('abortMultipartupload API failed for ' . 623 | $bucket . '/' . $object); 624 | 625 | $result = $s3Client->abortMultipartUpload([ 626 | 'Bucket' => $bucket, 627 | 'Key' => $object, 628 | 'UploadId' => 'non-existent', 629 | ]); 630 | 631 | if (getStatusCode($result) != HTTP_NOCONTENT) 632 | throw new Exception('abortMultipartupload API failed for ' . 633 | $bucket . '/' . $object); 634 | } 635 | 636 | /** 637 | * testGetBucketLocation tests GET bucket location S3 API 638 | * 639 | * @param $s3Client AWS\S3\S3Client object 640 | * 641 | * @param $params associative array containing bucket name 642 | * 643 | * @return void 644 | */ 645 | function testGetBucketLocation($s3Client, $params) { 646 | $bucket = $params['Bucket']; 647 | 648 | // Valid test 649 | $result = $s3Client->getBucketLocation(['Bucket' => $bucket]); 650 | if (getStatusCode($result) != HTTP_OK) 651 | throw new Exception('getBucketLocation API failed for ' . 652 | $bucket); 653 | 654 | // Run failure tests. 655 | $params = [ 656 | // Non existent bucket 657 | 'NoSuchBucket' => ['Bucket' => $bucket . '--'], 658 | 659 | // Bucket not found 660 | 'NoSuchBucket' => ['Bucket' => $bucket . '-non-existent'], 661 | ]; 662 | runExceptionalTests($s3Client, 'getBucketLocation', 'getAwsErrorCode', $params); 663 | } 664 | 665 | /** 666 | * testCopyObject tests copy object S3 API 667 | * 668 | * @param $s3Client AWS\S3\S3Client object 669 | * 670 | * @param $params associative array containing bucket and object name 671 | * 672 | * @return void 673 | */ 674 | function testCopyObject($s3Client, $params) { 675 | $bucket = $params['Bucket']; 676 | $object = $params['Object']; 677 | 678 | $result = $s3Client->copyObject([ 679 | 'Bucket' => $bucket, 680 | 'Key' => $object . '-copy', 681 | 'CopySource' => $bucket . '/' . $object, 682 | ]); 683 | if (getStatusCode($result) != HTTP_OK) 684 | throw new Exception('copyObject API failed for ' . 685 | $bucket); 686 | 687 | $s3Client->deleteObject([ 688 | 'Bucket' => $bucket, 689 | 'Key' => $object . '-copy', 690 | ]); 691 | 692 | // Run failure tests 693 | $params = [ 694 | // Invalid copy source format 695 | 'InvalidArgument' => [ 696 | 'Bucket' => $bucket, 697 | 'Key' => $object . '-copy', 698 | 'CopySource' => $bucket . $object 699 | ], 700 | 701 | // Missing source object 702 | 'NoSuchKey' => [ 703 | 'Bucket' => $bucket, 704 | 'Key' => $object . '-copy', 705 | 'CopySource' => $bucket . '/' . $object . '-non-existent' 706 | ], 707 | ]; 708 | runExceptionalTests($s3Client, 'copyObject', 'getAwsErrorCode', $params); 709 | } 710 | 711 | /** 712 | * testDeleteObjects tests Delete Objects S3 API 713 | * 714 | * @param $s3Client AWS\S3\S3Client object 715 | * 716 | * @param $params associative array containing bucket and object names 717 | * 718 | * @return void 719 | */ 720 | function testDeleteObjects($s3Client, $params) { 721 | $bucket = $params['Bucket']; 722 | $object = $params['Object']; 723 | 724 | $copies = []; 725 | for ($i = 0; $i < 3; $i++) { 726 | $copyKey = $object . '-copy' . strval($i); 727 | $result = $s3Client->copyObject([ 728 | 'Bucket' => $bucket, 729 | 'Key' => $copyKey, 730 | 'CopySource' => $bucket . '/' . $object, 731 | ]); 732 | if (getstatuscode($result) != HTTP_OK) 733 | throw new Exception('copyobject API failed for ' . 734 | $bucket); 735 | array_push($copies, ['Key' => $copyKey]); 736 | } 737 | 738 | $result = $s3Client->deleteObjects([ 739 | 'Bucket' => $bucket, 740 | 'Delete' => [ 741 | 'Objects' => $copies, 742 | ], 743 | ]); 744 | if (getstatuscode($result) != HTTP_OK) 745 | throw new Exception('deleteObjects api failed for ' . 746 | $bucket); 747 | } 748 | 749 | // Check if the policy statements are equal 750 | function are_statements_equal($expected, $got) { 751 | $expected = json_decode($expected, TRUE); 752 | $got = json_decode($got, TRUE); 753 | 754 | function are_actions_equal($action1, $action2) { 755 | return ( 756 | is_array($action1) 757 | && is_array($action2) 758 | && count($action1) == count($action2) 759 | && array_diff($action1, $action2) === array_diff($action2, $action1) 760 | ); 761 | } 762 | 763 | foreach ($expected['Statement'] as $index => $value) { 764 | if (!are_actions_equal($value['Action'], $got['Statement'][$index]['Action'])) 765 | return FALSE; 766 | } 767 | 768 | return TRUE; 769 | 770 | } 771 | /** 772 | * testBucketPolicy tests GET/PUT/DELETE Bucket policy S3 APIs 773 | * 774 | * @param $s3Client AWS\S3\S3Client object 775 | * 776 | * @param $params associative array containing bucket and object names 777 | * 778 | * @return void 779 | */ 780 | function testBucketPolicy($s3Client, $params) { 781 | $bucket = $params['Bucket']; 782 | 783 | $downloadPolicy = sprintf('{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetBucketLocation","s3:ListBucket","s3:GetObject"],"Resource":["arn:aws:s3:::%s","arn:aws:s3:::%s/*"]}]}', $bucket, $bucket); 784 | 785 | $result = $s3Client->putBucketPolicy([ 786 | 'Bucket' => $bucket, 787 | 'Policy' => $downloadPolicy 788 | ]); 789 | if (getstatuscode($result) != HTTP_NOCONTENT) 790 | throw new Exception('putBucketPolicy API failed for ' . 791 | $bucket); 792 | $result = $s3Client->getBucketPolicy(['Bucket' => $bucket]); 793 | if (getstatuscode($result) != HTTP_OK) 794 | throw new Exception('getBucketPolicy API failed for ' . 795 | $bucket); 796 | 797 | if ($result['Policy'] != $downloadPolicy) 798 | if (!are_statements_equal($result['Policy'], $downloadPolicy)) 799 | throw new Exception('bucket policy we got is not we set'); 800 | 801 | $result = $s3Client->getBucketPolicyStatus(['Bucket' => $bucket]); 802 | $result = $result->get("PolicyStatus")["IsPublic"]; 803 | if ($result) 804 | throw new Exception('getBucketPolicyStatus API failed for ' . 805 | $bucket); 806 | 807 | // Delete the bucket, make the bucket (again) and check if policy is none 808 | // Ref: https://github.com/minio/minio/issues/4714 809 | $result = $s3Client->deleteBucket(['Bucket' => $bucket]); 810 | if (getstatuscode($result) != HTTP_NOCONTENT) 811 | throw new Exception('deleteBucket API failed for ' . 812 | $bucket); 813 | 814 | try { 815 | $s3Client->getBucketPolicy(['Bucket' => $bucket]); 816 | } catch (AWSException $e) { 817 | switch ($e->getAwsErrorCode()) { 818 | case 'NoSuchBucket': 819 | break; 820 | } 821 | } 822 | 823 | // Sleep is needed for Minio Gateway for Azure, ref: 824 | // https://docs.microsoft.com/en-us/rest/api/storageservices/Delete-Container#remarks 825 | sleep(40); 826 | 827 | $s3Client->createBucket(['Bucket' => $bucket]); 828 | 829 | $params = [ 830 | '404' => ['Bucket' => $bucket] 831 | ]; 832 | runExceptionalTests($s3Client, 'getBucketPolicy', 'getStatusCode', $params); 833 | 834 | try { 835 | $MINT_DATA_DIR = $GLOBALS['MINT_DATA_DIR']; 836 | // Create an object to test anonymous GET object 837 | $object = 'test-anon'; 838 | if (!file_exists($MINT_DATA_DIR . '/' . FILE_1_KB)) 839 | throw new Exception('File not found ' . $MINT_DATA_DIR . '/' . FILE_1_KB); 840 | 841 | $stream = Psr7\Utils::streamFor(fopen($MINT_DATA_DIR . '/' . FILE_1_KB, 'r')); 842 | $result = $s3Client->putObject([ 843 | 'Bucket' => $bucket, 844 | 'Key' => $object, 845 | 'Body' => $stream, 846 | ]); 847 | if (getstatuscode($result) != HTTP_OK) 848 | throw new Exception('createBucket API failed for ' . 849 | $bucket); 850 | 851 | $anonConfig = new ClientConfig("", "", $GLOBALS['endpoint'], $GLOBALS['secure'], $GLOBALS['region']); 852 | $anonymousClient = new S3Client([ 853 | 'credentials' => false, 854 | 'endpoint' => $anonConfig->endpoint, 855 | 'use_path_style_endpoint' => true, 856 | 'region' => $anonConfig->region, 857 | 'version' => '2006-03-01' 858 | ]); 859 | runExceptionalTests($anonymousClient, 'getObject', 'getStatusCode', [ 860 | '403' => [ 861 | 'Bucket' => $bucket, 862 | 'Key' => $object, 863 | ] 864 | ]); 865 | 866 | $result = $s3Client->putBucketPolicy([ 867 | 'Bucket' => $bucket, 868 | 'Policy' => $downloadPolicy 869 | ]); 870 | if (getstatuscode($result) != HTTP_NOCONTENT) 871 | throw new Exception('putBucketPolicy API failed for ' . 872 | $bucket); 873 | $result = $s3Client->getBucketPolicy(['Bucket' => $bucket]); 874 | if (getstatuscode($result) != HTTP_OK) 875 | throw new Exception('getBucketPolicy API failed for ' . 876 | $bucket); 877 | 878 | $result = $s3Client->deleteBucketPolicy(['Bucket' => $bucket]); 879 | if (getstatuscode($result) != HTTP_NOCONTENT) 880 | throw new Exception('deleteBucketPolicy API failed for ' . 881 | $bucket); 882 | } finally { 883 | // close data file 884 | if (!is_null($stream)) 885 | $stream->close(); 886 | $s3Client->deleteObject(['Bucket' => $bucket, 'Key' => $object]); 887 | } 888 | 889 | } 890 | 891 | /** 892 | * cleanupSetup removes all buckets and objects created during the 893 | * functional test 894 | * 895 | * @param $s3Client AWS\S3\S3Client object 896 | * 897 | * @param $objects Associative array of buckets to objects 898 | * 899 | * @return void 900 | */ 901 | function cleanupSetup($s3Client, $objects) { 902 | // Delete all objects 903 | foreach ($objects as $bucket => $object) { 904 | $s3Client->deleteObject(['Bucket' => $bucket, 'Key' => $object]); 905 | } 906 | 907 | // Delete the buckets incl. emptyBucket 908 | $allBuckets = array_keys($objects); 909 | array_push($allBuckets, $GLOBALS['emptyBucket']); 910 | foreach ($allBuckets as $bucket) { 911 | try { 912 | // Delete the bucket 913 | $s3Client->deleteBucket(['Bucket' => $bucket]); 914 | 915 | // Wait until the bucket is removed from object store 916 | $s3Client->waitUntil('BucketNotExists', ['Bucket' => $bucket]); 917 | } catch (Exception $e) { 918 | // Ignore exceptions thrown during cleanup 919 | } 920 | } 921 | } 922 | 923 | 924 | /** 925 | * runTest helper function to wrap a test function and log 926 | * success or failure accordingly. 927 | * 928 | * @param myfunc name of test function to be run 929 | * 930 | * @param fnSignature function signature of the main S3 SDK API 931 | * 932 | * @param args parameters to be passed to test function 933 | * 934 | * @return void 935 | */ 936 | function runTest($s3Client, $myfunc, $fnSignature, $args = []) { 937 | try { 938 | $start_time = microtime(true); 939 | $status = "PASS"; 940 | $error = ""; 941 | $message = ""; 942 | $myfunc($s3Client, $args); 943 | } catch (AwsException $e) { 944 | $errorCode = $e->getAwsErrorCode(); 945 | // $fnSignature holds the specific API that is being 946 | // tested. It is possible that functions used to create the 947 | // test setup may not be implemented. 948 | if ($errorCode != "NotImplemented") { 949 | $status = "FAIL"; 950 | $error = $e->getMessage(); 951 | } else { 952 | $status = "NA"; 953 | $error = $e->getMessage(); 954 | $alert = sprintf("%s or a related API is NOT IMPLEMENTED, see \"error\" for exact details.", $fnSignature); 955 | } 956 | 957 | } catch (Exception $e) { 958 | // This exception handler handles high-level custom exceptions. 959 | $status = "FAIL"; 960 | $error = $e->getMessage(); 961 | } finally { 962 | $end_time = microtime(true); 963 | $json_log = [ 964 | "name" => "aws-sdk-php", 965 | "function" => $fnSignature, 966 | "args" => $args, 967 | "duration" => sprintf("%d", ($end_time - $start_time) * 1000), // elapsed time in ms 968 | "status" => $status, 969 | ]; 970 | if ($error !== "") { 971 | $json_log["error"] = $error; 972 | } 973 | if ($message !== "") { 974 | $json_log["message"] = $message; 975 | } 976 | print_r(json_encode($json_log)."\n"); 977 | 978 | // Exit on first failure. 979 | switch ($status) { 980 | case "FAIL": 981 | exit(1); 982 | } 983 | } 984 | } 985 | 986 | // Get client configuration from environment variables 987 | $GLOBALS['access_key'] = getenv("ACCESS_KEY"); 988 | $GLOBALS['secret_key'] = getenv("SECRET_KEY"); 989 | $GLOBALS['endpoint'] = getenv("SERVER_ENDPOINT"); 990 | $GLOBALS['region'] = getenv("SERVER_REGION"); 991 | $GLOBALS['secure'] = getenv("ENABLE_HTTPS"); 992 | 993 | /** 994 | * @global string $GLOBALS['MINT_DATA_DIR'] 995 | * @name $MINT_DATA_DIR 996 | */ 997 | $GLOBALS['MINT_DATA_DIR'] = '/mint/data'; 998 | $GLOBALS['MINT_DATA_DIR'] = getenv("MINT_DATA_DIR"); 999 | 1000 | 1001 | // Useful for debugging test failures; Set $debugmode it to true when required 1002 | $debugmode = false; 1003 | 1004 | interface Debugger { 1005 | public function out($data); 1006 | } 1007 | 1008 | class EchoDebugger implements Debugger { 1009 | public function out($data) { 1010 | echo $data; 1011 | } 1012 | } 1013 | 1014 | class NullDebugger implements Debugger { 1015 | public function out($data) { 1016 | // Do nothing 1017 | } 1018 | } 1019 | 1020 | if($debugmode) 1021 | $debugger = new EchoDebugger(); 1022 | else 1023 | $debugger = new NullDebugger(); 1024 | 1025 | // Make $debugger global 1026 | $GLOBALS['debugger'] = $debugger; 1027 | 1028 | // Create config object 1029 | $config = new ClientConfig($GLOBALS['access_key'], $GLOBALS['secret_key'], 1030 | $GLOBALS['endpoint'], $GLOBALS['secure'], 1031 | $GLOBALS['region']); 1032 | 1033 | // Create a S3Client 1034 | $s3Client = new S3Client([ 1035 | 'credentials' => $config->creds, 1036 | 'endpoint' => $config->endpoint, 1037 | 'use_path_style_endpoint' => true, 1038 | 'region' => $config->region, 1039 | 'version' => '2006-03-01' 1040 | ]); 1041 | 1042 | // Used by initSetup 1043 | $emptyBucket = randomName(); 1044 | $objects = [ 1045 | randomName() => 'obj1', 1046 | randomName() => 'obj2', 1047 | ]; 1048 | 1049 | try { 1050 | initSetup($s3Client, $objects); 1051 | $firstBucket = array_keys($objects)[0]; 1052 | $firstObject = $objects[$firstBucket]; 1053 | $testParams = ['Bucket' => $firstBucket, 'Object' => $firstObject]; 1054 | runTest($s3Client, 'testGetBucketLocation', "getBucketLocation ( array \$params = [] )", ['Bucket' => $firstBucket]); 1055 | runTest($s3Client, 'testListBuckets', "listBuckets ( array \$params = [] )"); 1056 | runTest($s3Client, 'testListObjects', "listObjects ( array \$params = [] )", $testParams); 1057 | runTest($s3Client, 'testListMultipartUploads', "listMultipartUploads ( array \$params = [] )", $testParams); 1058 | runTest($s3Client, 'testBucketExists', "headBucket ( array \$params = [] )", array_keys($objects)); 1059 | runTest($s3Client, 'testHeadObject', "headObject ( array \$params = [] )", $objects); 1060 | runTest($s3Client, 'testGetPutObject', "getObject ( array \$params = [] )", $testParams); 1061 | runTest($s3Client, 'testCopyObject', "copyObject ( array \$params = [] )", $testParams); 1062 | runTest($s3Client, 'testDeleteObjects', "deleteObjects (array \$params = [] )", $testParams); 1063 | runTest($s3Client, 'testMultipartUpload', "createMultipartUpload ( array \$params = [] )", $testParams); 1064 | runTest($s3Client, 'testMultipartUploadFailure', "uploadPart ( array \$params = [] )", $testParams); 1065 | runTest($s3Client, 'testAbortMultipartUpload', "abortMultipartupload ( array \$params = [] )", $testParams); 1066 | runTest($s3Client, 'testBucketPolicy', "getBucketPolicy ( array \$params = [] )", ['Bucket' => $emptyBucket]); 1067 | } 1068 | finally { 1069 | cleanupSetup($s3Client, $objects); 1070 | } 1071 | 1072 | ?> 1073 | --------------------------------------------------------------------------------