├── pre-commit ├── settings.gradle ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── ci ├── docker │ ├── entrypoint_legacy_systest.sh │ ├── entrypoint.sh │ ├── Dockerfile.scripting-jython │ ├── Dockerfile.scripting-groovy │ ├── Dockerfile.legacy-systest │ ├── Dockerfile.robotest │ └── Dockerfile └── .gitlab-ci.yml.old ├── .editorconfig ├── gradle.properties ├── .github └── workflows │ └── main.yml ├── .gitignore ├── LICENSE ├── gradlew.bat ├── gradlew ├── idea-codestyle.xml ├── README.md └── CONTRIBUTING.md /pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ./gradlew spotlessApply 4 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = "mongoose-bundle" 2 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/emc-mongoose/mongoose/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /ci/docker/entrypoint_legacy_systest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | umask 0000 3 | ./gradlew clean systemTest --tests com.emc.mongoose.system.${TEST} 4 | -------------------------------------------------------------------------------- /ci/docker/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | umask 0000 3 | export JAVA_HOME=/opt/mongoose 4 | export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${JAVA_HOME}/bin 5 | java -jar /opt/mongoose/mongoose.jar --storage-driver-type=s3 --storage-net-node-port=9020 "$@" 6 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | [*] 2 | charset=utf-8 3 | end_of_line=lf 4 | insert_final_newline=true 5 | indent_style=tab 6 | tab_width=4 7 | 8 | [{*.yml,*.yaml}] 9 | indent_style=space 10 | indent_size=2 11 | 12 | [{*.md,*.mkd,*.markdown}] 13 | indent_style=space 14 | indent_size=4 15 | 16 | -------------------------------------------------------------------------------- /ci/docker/Dockerfile.scripting-jython: -------------------------------------------------------------------------------- 1 | ARG MONGOOSE_VERSION 2 | 3 | FROM emcmongoose/mongoose:${MONGOOSE_VERSION} 4 | 5 | ADD ["http://central.maven.org/maven2/org/python/jython-standalone/2.7.1/jython-standalone-2.7.1.jar", "/root/.mongoose/${MONGOOSE_VERSION}/ext/jython-standalone.jar"] 6 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Sun Feb 03 10:01:50 MSK 2019 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.3-all.zip 7 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | baseVersion=4.3.16 2 | loadStepPipelineVersion=4.2.13 3 | loadStepWeightedVersion=4.2.12 4 | storageDriverCoopVersion=4.2.21 5 | storageDriverNettyVersion=4.2.21 6 | storageDriverNioVersion=4.2.14 7 | storageDriverHttpVersion=4.2.22 8 | storageDriverFsVersion=4.2.13 9 | storageDriverAtmosVersion=4.2.14 10 | storageDriverS3Version=4.2.23 11 | storageDriverSwiftVersion=4.2.14 12 | -------------------------------------------------------------------------------- /ci/docker/Dockerfile.scripting-groovy: -------------------------------------------------------------------------------- 1 | ARG MONGOOSE_VERSION 2 | 3 | FROM emcmongoose/mongoose:${MONGOOSE_VERSION} 4 | 5 | ADD ["http://central.maven.org/maven2/org/codehaus/groovy/groovy/2.4.15/groovy-2.4.15.jar", "/root/.mongoose/${MONGOOSE_VERSION}/ext/groovy.jar"] 6 | ADD ["http://central.maven.org/maven2/org/codehaus/groovy/groovy-jsr223/2.4.15/groovy-jsr223-2.4.15.jar", "/root/.mongoose/${MONGOOSE_VERSION}/ext/groovy-jsr223.jar"] 7 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | on: [push] 2 | jobs: 3 | build_job: 4 | uses: 5 | emc-mongoose/mongoose-base/.github/workflows/main.yml@master 6 | secrets: 7 | OSSRH_USERNAME: ${{ secrets.OSSRH_USERNAME }} 8 | OSSRH_PASSWORD: ${{ secrets.OSSRH_PASSWORD }} 9 | GPG_SECRET_KEY: ${{ secrets.GPG_SECRET_KEY }} 10 | GPG_SECRET_KEY_PASSWORD: ${{ secrets.GPG_SECRET_KEY_PASSWORD }} 11 | GPG_SECRING: ${{ secrets.GPG_SECRING }} 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # IntelliJ project files 2 | .idea 3 | *.iml 4 | out 5 | gen### Java template 6 | *.class 7 | 8 | # Package Files # 9 | *.jar 10 | *.war 11 | *.ear 12 | *.tgz 13 | build/* 14 | */build/* 15 | **/build/* 16 | mongoose-* 17 | MANIFEST.MF 18 | 19 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 20 | hs_err_pid* 21 | 22 | result*.xml 23 | 24 | # protect the gradle wrapper 25 | !gradle/wrapper/gradle-wrapper.jar 26 | -------------------------------------------------------------------------------- /ci/docker/Dockerfile.legacy-systest: -------------------------------------------------------------------------------- 1 | FROM docker:dind 2 | 3 | RUN apk add --no-cache --update openjdk8 4 | 5 | ADD ci /root/mongoose/ci 6 | ADD gradle /root/mongoose/gradle 7 | ADD load /root/mongoose/load 8 | ADD src /root/mongoose/src 9 | ADD storage /root/mongoose/storage 10 | ADD build.gradle /root/mongoose/build.gradle 11 | ADD gradlew /root/mongoose/gradlew 12 | ADD settings.gradle /root/mongoose/settings.gradle 13 | 14 | RUN chmod ugo+x /root/mongoose/ci/docker/entrypoint_legacy_systest.sh 15 | 16 | WORKDIR /root/mongoose 17 | 18 | ENTRYPOINT ["/root/mongoose/docker/entrypoint_legacy_systest.sh"] 19 | -------------------------------------------------------------------------------- /ci/docker/Dockerfile.robotest: -------------------------------------------------------------------------------- 1 | FROM docker:dind 2 | 3 | RUN apk add --no-cache --update python py-pip \ 4 | && pip install -U virtualenv \ 5 | && pip install -U requests \ 6 | && pip install -U robotframework \ 7 | && pip install -U robotframework-requests \ 8 | && pip install -U robotframework-csvlibrary 9 | 10 | ENV PYTHONPATH=$PYTHONPATH:/usr/lib/python2.7/site-packages:/root/mongoose/base/src/test/robot/lib 11 | 12 | ADD base/build /root/mongoose/base/build 13 | ADD ci/docker/entrypoint_robotest.sh /root/mongoose/ci/docker/entrypoint_robotest.sh 14 | ADD base/src/test/robot /root/mongoose/base/src/test/robot 15 | 16 | RUN chmod ugo+x /root/mongoose/ci/docker/entrypoint_robotest.sh 17 | 18 | WORKDIR /root/mongoose 19 | 20 | ENTRYPOINT ["/root/mongoose/ci/docker/entrypoint_robotest.sh"] 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Dell Corporation 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS="-Xmx64m" 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /ci/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG BASE_VERSION 2 | FROM emcmongoose/mongoose-base:${BASE_VERSION} 3 | ARG BASE_VERSION 4 | ARG LOAD_STEP_PIPELINE_VERSION 5 | ARG LOAD_STEP_WEIGHTED_VERSION 6 | ARG STORAGE_DRIVER_COOP_VERSION 7 | ARG STORAGE_DRIVER_NETTY_VERSION 8 | ARG STORAGE_DRIVER_NIO_VERSION 9 | ARG STORAGE_DRIVER_FS_VERSION 10 | ARG STORAGE_DRIVER_HTTP_VERSION 11 | ARG STORAGE_DRIVER_ATMOS_VERSION 12 | ARG STORAGE_DRIVER_S3_VERSION 13 | ARG STORAGE_DRIVER_SWIFT_VERSION 14 | ADD ci/docker/entrypoint.sh /opt/mongoose/entrypoint.sh 15 | RUN mkdir -p $HOME/.mongoose/${BASE_VERSION}/ext \ 16 | && chmod +x /opt/mongoose/entrypoint.sh \ 17 | && curl https://repo.maven.apache.org/maven2/com/github/emc-mongoose/mongoose-load-step-pipeline/${LOAD_STEP_PIPELINE_VERSION}/mongoose-load-step-pipeline-${LOAD_STEP_PIPELINE_VERSION}.jar -o $HOME/.mongoose/${BASE_VERSION}/ext/mongoose-load-step-pipeline.jar \ 18 | && curl https://repo.maven.apache.org/maven2/com/github/emc-mongoose/mongoose-load-step-weighted/${LOAD_STEP_WEIGHTED_VERSION}/mongoose-load-step-weighted-${LOAD_STEP_WEIGHTED_VERSION}.jar -o $HOME/.mongoose/${BASE_VERSION}/ext/mongoose-load-step-weighted.jar \ 19 | && curl https://repo.maven.apache.org/maven2/com/github/emc-mongoose/mongoose-storage-driver-coop/${STORAGE_DRIVER_COOP_VERSION}/mongoose-storage-driver-coop-${STORAGE_DRIVER_COOP_VERSION}.jar -o $HOME/.mongoose/${BASE_VERSION}/ext/mongoose-storage-driver-coop.jar \ 20 | && curl https://repo.maven.apache.org/maven2/com/github/emc-mongoose/mongoose-storage-driver-netty/${STORAGE_DRIVER_NETTY_VERSION}/mongoose-storage-driver-netty-${STORAGE_DRIVER_NETTY_VERSION}.jar -o $HOME/.mongoose/${BASE_VERSION}/ext/mongoose-storage-driver-netty.jar \ 21 | && curl https://repo.maven.apache.org/maven2/com/github/emc-mongoose/mongoose-storage-driver-nio/${STORAGE_DRIVER_NIO_VERSION}/mongoose-storage-driver-nio-${STORAGE_DRIVER_NIO_VERSION}.jar -o $HOME/.mongoose/${BASE_VERSION}/ext/mongoose-storage-driver-nio.jar \ 22 | && curl https://repo.maven.apache.org/maven2/com/github/emc-mongoose/mongoose-storage-driver-fs/${STORAGE_DRIVER_FS_VERSION}/mongoose-storage-driver-fs-${STORAGE_DRIVER_FS_VERSION}.jar -o $HOME/.mongoose/${BASE_VERSION}/ext/mongoose-storage-driver-fs.jar \ 23 | && curl https://repo.maven.apache.org/maven2/com/github/emc-mongoose/mongoose-storage-driver-http/${STORAGE_DRIVER_HTTP_VERSION}/mongoose-storage-driver-http-${STORAGE_DRIVER_HTTP_VERSION}.jar -o $HOME/.mongoose/${BASE_VERSION}/ext/mongoose-storage-driver-http.jar \ 24 | && curl https://repo.maven.apache.org/maven2/com/github/emc-mongoose/mongoose-storage-driver-atmos/${STORAGE_DRIVER_ATMOS_VERSION}/mongoose-storage-driver-atmos-${STORAGE_DRIVER_ATMOS_VERSION}.jar -o $HOME/.mongoose/${BASE_VERSION}/ext/mongoose-storage-driver-atmos.jar \ 25 | && curl https://repo.maven.apache.org/maven2/com/github/emc-mongoose/mongoose-storage-driver-s3/${STORAGE_DRIVER_S3_VERSION}/mongoose-storage-driver-s3-${STORAGE_DRIVER_S3_VERSION}.jar -o $HOME/.mongoose/${BASE_VERSION}/ext/mongoose-storage-driver-s3.jar \ 26 | && curl https://repo.maven.apache.org/maven2/com/github/emc-mongoose/mongoose-storage-driver-swift/${STORAGE_DRIVER_SWIFT_VERSION}/mongoose-storage-driver-swift-${STORAGE_DRIVER_SWIFT_VERSION}.jar -o $HOME/.mongoose/${BASE_VERSION}/ext/mongoose-storage-driver-swift.jar 27 | -------------------------------------------------------------------------------- /ci/.gitlab-ci.yml.old: -------------------------------------------------------------------------------- 1 | image: docker:stable 2 | 3 | variables: 4 | DOCKER_HOST: tcp://docker:2375 5 | DOCKER_DRIVER: overlay2 6 | IMAGE_NAME: emcmongoose/mongoose 7 | IMAGE_FILE_NAME: build/mongoose.tar 8 | JAVA_HOME: /opt/jdk-11.0.2+9 9 | ROBOTEST_CONTAINER_WORKING_DIR: /root/mongoose 10 | SERVICE_HOST: docker # should be used instead of the "localhost"/"127.0.0.1" in GL CI 11 | 12 | services: 13 | - docker:dind 14 | 15 | stages: 16 | - build 17 | - build_docker_image 18 | - test_functional # functional containerized tests 19 | - deploy 20 | 21 | before_script: 22 | - apk --update add --no-cache ca-certificates curl openssl binutils xz gnupg 23 | - curl -Lks https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.2%2B9/OpenJDK11U-jdk_x64_linux_hotspot_11.0.2_9.tar.gz -o /tmp/jdk11.tgz 24 | - tar xvf /tmp/jdk11.tgz -C /opt 25 | - rm -rf /tmp/jdk11.tgz 26 | - export PATH=${PATH}:${JAVA_HOME}/bin 27 | - export GLIBC_VER="2.28-r0" 28 | - export ALPINE_GLIBC_REPO="https://github.com/sgerrand/alpine-pkg-glibc/releases/download" 29 | - export GCC_LIBS_URL="https://archive.archlinux.org/packages/g/gcc-libs/gcc-libs-8.3.0-1-x86_64.pkg.tar.xz" 30 | - export GCC_LIBS_SHA256=400e2ecb1b2dfb40e09cdb6805f0075cbc88e6fcef9b73f23c64a6e709dcd61b 31 | - export ZLIB_URL="https://archive.archlinux.org/packages/z/zlib/zlib-1%3A1.2.11-3-x86_64.pkg.tar.xz" 32 | - export ZLIB_SHA256=17aede0b9f8baa789c5aa3f358fbf8c68a5f1228c5e6cba1a5dd34102ef4d4e5 33 | - curl -Lks https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub -o /etc/apk/keys/sgerrand.rsa.pub 34 | - curl -Lks ${ALPINE_GLIBC_REPO}/${GLIBC_VER}/glibc-${GLIBC_VER}.apk > /tmp/${GLIBC_VER}.apk 35 | - apk add --allow-untrusted /tmp/${GLIBC_VER}.apk 36 | - curl -Lks ${GCC_LIBS_URL} -o /tmp/gcc-libs.tar.xz 37 | - echo "${GCC_LIBS_SHA256} /tmp/gcc-libs.tar.xz" | sha256sum -c - 38 | - mkdir /tmp/gcc 39 | - tar -xf /tmp/gcc-libs.tar.xz -C /tmp/gcc 40 | - mv /tmp/gcc/usr/lib/libgcc* /tmp/gcc/usr/lib/libstdc++* /usr/glibc-compat/lib 41 | - strip /usr/glibc-compat/lib/libgcc_s.so.* /usr/glibc-compat/lib/libstdc++.so* 42 | - curl -Lks ${ZLIB_URL} -o /tmp/libz.tar.xz 43 | - echo "${ZLIB_SHA256} /tmp/libz.tar.xz" | sha256sum -c - 44 | - mkdir /tmp/libz 45 | - tar -xf /tmp/libz.tar.xz -C /tmp/libz 46 | - mv /tmp/libz/usr/lib/libz.so* /usr/glibc-compat/lib 47 | - apk del binutils 48 | - rm -rf /tmp/${GLIBC_VER}.apk /tmp/gcc /tmp/gcc-libs.tar.xz /tmp/libz /tmp/libz.tar.xz /var/cache/apk/* 49 | 50 | build: 51 | stage: build 52 | script: 53 | - ./gradlew clean jar 54 | artifacts: 55 | paths: 56 | - build/libs/mongoose-*.jar 57 | 58 | build_docker_image: 59 | stage: build_docker_image 60 | variables: 61 | DOCKERFILE: Dockerfile 62 | script: 63 | - ./gradlew dockerBuildImage 64 | - export VERSION=$(cat build.gradle | grep version\ = | sed -n 's/.*\([0-9]\+\.[0-9]\+\.[0-9]\+\).*/\1/p') 65 | - docker save ${IMAGE_NAME}:${VERSION} > ${IMAGE_FILE_NAME} 66 | artifacts: 67 | paths: 68 | - ${IMAGE_FILE_NAME} 69 | 70 | release_to_maven_repo: 71 | stage: deploy 72 | script: 73 | - if [ ! -z "$GPG_SECRET_KEYS" ]; then echo $GPG_SECRET_KEYS | base64 -d | gpg --import --batch; fi 74 | - if [ ! -z "$GPG_OWNERTRUST" ]; then echo $GPG_OWNERTRUST | base64 -d | gpg --import-ownertrust --batch; fi 75 | - mkdir /tmp/.gnupg 76 | - if [ ! -z "$GPG_SECRING" ]; then echo $GPG_SECRING | base64 -d > /tmp/.gnupg/secring.gpg; fi 77 | - ./gradlew -Psigning.keyId=${SIGNING_KEY_ID} -Psigning.password=${SIGNING_PASSWORD} -Psigning.secretKeyRingFile=/tmp/.gnupg/secring.gpg -PossrhUsername=${OSSRH_USERNAME} -PossrhPassword=${OSSRH_PASSWORD} publishToNexus closeAndReleaseRepository 78 | only: 79 | - latest 80 | except: 81 | - branches 82 | 83 | deploy_to_docker_hub: 84 | stage: deploy 85 | script: 86 | - docker login -u ${DOCKER_USER} -p ${DOCKER_PASS} 87 | - docker load < ${IMAGE_FILE_NAME} 88 | - export VERSION=$(cat build.gradle | grep version\ = | sed -n 's/.*\([0-9]\+\.[0-9]\+\.[0-9]\+\).*/\1/p') 89 | - docker push ${IMAGE_NAME}:${VERSION} 90 | - docker tag ${IMAGE_NAME}:${VERSION} ${IMAGE_NAME}:latest 91 | - docker push ${IMAGE_NAME}:latest 92 | only: 93 | - latest 94 | except: 95 | - branches 96 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Attempt to set APP_HOME 10 | # Resolve links: $0 may be a link 11 | PRG="$0" 12 | # Need this for relative symlinks. 13 | while [ -h "$PRG" ] ; do 14 | ls=`ls -ld "$PRG"` 15 | link=`expr "$ls" : '.*-> \(.*\)$'` 16 | if expr "$link" : '/.*' > /dev/null; then 17 | PRG="$link" 18 | else 19 | PRG=`dirname "$PRG"`"/$link" 20 | fi 21 | done 22 | SAVED="`pwd`" 23 | cd "`dirname \"$PRG\"`/" >/dev/null 24 | APP_HOME="`pwd -P`" 25 | cd "$SAVED" >/dev/null 26 | 27 | APP_NAME="Gradle" 28 | APP_BASE_NAME=`basename "$0"` 29 | 30 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 31 | DEFAULT_JVM_OPTS='"-Xmx64m"' 32 | 33 | # Use the maximum available, or set MAX_FD != -1 to use that value. 34 | MAX_FD="maximum" 35 | 36 | warn () { 37 | echo "$*" 38 | } 39 | 40 | die () { 41 | echo 42 | echo "$*" 43 | echo 44 | exit 1 45 | } 46 | 47 | # OS specific support (must be 'true' or 'false'). 48 | cygwin=false 49 | msys=false 50 | darwin=false 51 | nonstop=false 52 | case "`uname`" in 53 | CYGWIN* ) 54 | cygwin=true 55 | ;; 56 | Darwin* ) 57 | darwin=true 58 | ;; 59 | MINGW* ) 60 | msys=true 61 | ;; 62 | NONSTOP* ) 63 | nonstop=true 64 | ;; 65 | esac 66 | 67 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 68 | 69 | # Determine the Java command to use to start the JVM. 70 | if [ -n "$JAVA_HOME" ] ; then 71 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 72 | # IBM's JDK on AIX uses strange locations for the executables 73 | JAVACMD="$JAVA_HOME/jre/sh/java" 74 | else 75 | JAVACMD="$JAVA_HOME/bin/java" 76 | fi 77 | if [ ! -x "$JAVACMD" ] ; then 78 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 79 | 80 | Please set the JAVA_HOME variable in your environment to match the 81 | location of your Java installation." 82 | fi 83 | else 84 | JAVACMD="java" 85 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 86 | 87 | Please set the JAVA_HOME variable in your environment to match the 88 | location of your Java installation." 89 | fi 90 | 91 | # Increase the maximum file descriptors if we can. 92 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 93 | MAX_FD_LIMIT=`ulimit -H -n` 94 | if [ $? -eq 0 ] ; then 95 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 96 | MAX_FD="$MAX_FD_LIMIT" 97 | fi 98 | ulimit -n $MAX_FD 99 | if [ $? -ne 0 ] ; then 100 | warn "Could not set maximum file descriptor limit: $MAX_FD" 101 | fi 102 | else 103 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 104 | fi 105 | fi 106 | 107 | # For Darwin, add options to specify how the application appears in the dock 108 | if $darwin; then 109 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 110 | fi 111 | 112 | # For Cygwin, switch paths to Windows format before running java 113 | if $cygwin ; then 114 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 115 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 116 | JAVACMD=`cygpath --unix "$JAVACMD"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Escape application args 158 | save () { 159 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 160 | echo " " 161 | } 162 | APP_ARGS=$(save "$@") 163 | 164 | # Collect all arguments for the java command, following the shell quoting and substitution rules 165 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 166 | 167 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong 168 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then 169 | cd "$(dirname "$0")" 170 | fi 171 | 172 | exec "$JAVACMD" "$@" 173 | -------------------------------------------------------------------------------- /idea-codestyle.xml: -------------------------------------------------------------------------------- 1 | 2 | 9 | 142 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Maven metadata URL](https://img.shields.io/maven-metadata/v/http/central.maven.org/maven2/com/github/emc-mongoose/mongoose-bundle/maven-metadata.xml.svg?label=%20) 2 | [![CI status](https://gitlab.com/emc-mongoose/mongoose/badges/master/pipeline.svg)](https://gitlab.com/emc-mongoose/mongoose/commits/master) 3 | [![Docker Pulls](https://img.shields.io/docker/pulls/emcmongoose/mongoose.svg)](https://hub.docker.com/r/emcmongoose/mongoose/) 4 | [![Gitter chat](https://badges.gitter.im/emc-mongoose.png)](https://gitter.im/emc-mongoose) 5 | 6 | # Mongoose Bundle 7 | 8 | The repo contains the automation scripts to build/test/deploy the Mongoose backward compatibility bundle. Previously the 9 | repo contained the Mongoose sources for the basic functionality and some commonly used extensions. Currently it was 10 | split into the independent repos and the corresponding components. Each component has its own documentation, CI versioning. For the 11 | mongoose documentation refer this [link](https://github.com/emc-mongoose/mongoose-base/tree/master/doc). 12 | 13 | # Bundle Contents 14 | 15 | The components listed below are included in this backward compatibility bundle. 16 | 17 | | Repo | Description | Latest Release | Integration Status | Issue Tracker Link | 18 | |------|-------------|---------|-------------------------------|--------| 19 | | [mongoose-**base**](https://github.com/emc-mongoose/mongoose-base) | Mongoose storage performance testing tool - base functionality | ![Maven metadata URL](https://img.shields.io/maven-metadata/v/http/central.maven.org/maven2/com/github/emc-mongoose/mongoose-base/maven-metadata.xml.svg?label=%20) | ![Gitlab pipeline status](https://img.shields.io/gitlab/pipeline/emc-mongoose/mongoose-base.svg?label=%20) | [BASE](https://mongoose-issues.atlassian.net/projects/BASE) 20 | | [mongoose-load-step-**pipeline**](https://github.com/emc-mongoose/mongoose-load-step-pipeline) | Load operations pipeline (create,delay,read-then-update, for example), extension | ![Maven metadata URL](https://img.shields.io/maven-metadata/v/http/central.maven.org/maven2/com/github/emc-mongoose/mongoose-load-step-pipeline/maven-metadata.xml.svg?label=%20) | ![Gitlab pipeline status](https://img.shields.io/gitlab/pipeline/emc-mongoose/mongoose-load-step-pipeline.svg?label=%20) | [BASE](https://mongoose-issues.atlassian.net/projects/BASE) 21 | | [mongoose-load-step-**weighted**](https://github.com/emc-mongoose/mongoose-load-step-weighted) | Weighted load extension, allowing to generate 20% write and 80% read operations, for example | ![Maven metadata URL](https://img.shields.io/maven-metadata/v/http/central.maven.org/maven2/com/github/emc-mongoose/mongoose-load-step-weighted/maven-metadata.xml.svg?label=%20) | ![Gitlab pipeline status](https://img.shields.io/gitlab/pipeline/emc-mongoose/mongoose-load-step-weighted.svg?label=%20) | [BASE](https://mongoose-issues.atlassian.net/projects/BASE) 22 | | [mongoose-storage-driver-**coop**](https://github.com/emc-mongoose/mongoose-storage-driver-coop) | Cooperative multitasking storage driver primitive, utilizing [fibers](https://github.com/akurilov/fiber4j) | ![Maven metadata URL](https://img.shields.io/maven-metadata/v/http/central.maven.org/maven2/com/github/emc-mongoose/mongoose-storage-driver-coop/maven-metadata.xml.svg?label=%20) | ![Gitlab pipeline status](https://img.shields.io/gitlab/pipeline/emc-mongoose/mongoose-storage-driver-coop.svg?label=%20) | [BASE](https://mongoose-issues.atlassian.net/projects/BASE) 23 | | [mongoose-storage-driver-**netty**](https://github.com/emc-mongoose/mongoose-storage-driver-netty) | [Netty](https://netty.io/)-storage-driver-nettyd storage driver primitive, extends the cooperative multitasking storage driver primitive | ![Maven metadata URL](https://img.shields.io/maven-metadata/v/http/central.maven.org/maven2/com/github/emc-mongoose/mongoose-storage-driver-netty/maven-metadata.xml.svg?label=%20) | ![Gitlab pipeline status](https://img.shields.io/gitlab/pipeline/emc-mongoose/mongoose-storage-driver-netty.svg?label=%20) | [BASE](https://mongoose-issues.atlassian.net/projects/BASE) 24 | | [mongoose-storage-driver-**nio**](https://github.com/emc-mongoose/mongoose-storage-driver-nio) | Non-blocking I/O storage driver primitive, extends the cooperative multitasking storage driver primitive | ![Maven metadata URL](https://img.shields.io/maven-metadata/v/http/central.maven.org/maven2/com/github/emc-mongoose/mongoose-storage-driver-nio/maven-metadata.xml.svg?label=%20) | ![Gitlab pipeline status](https://img.shields.io/gitlab/pipeline/emc-mongoose/mongoose-storage-driver-nio.svg?label=%20) | [BASE](https://mongoose-issues.atlassian.net/projects/BASE) 25 | | [mongoose-storage-driver-**http**](https://github.com/emc-mongoose/mongoose-storage-driver-http) | HTTP storage driver primitive, extends the Netty-storage-driver-httpd storage driver primitive | ![Maven metadata URL](https://img.shields.io/maven-metadata/v/http/central.maven.org/maven2/com/github/emc-mongoose/mongoose-storage-driver-http/maven-metadata.xml.svg?label=%20) | ![Gitlab pipeline status](https://img.shields.io/gitlab/pipeline/emc-mongoose/mongoose-storage-driver-http.svg?label=%20) | [BASE](https://mongoose-issues.atlassian.net/projects/BASE) 26 | | [mongoose-storage-driver-**fs**](https://github.com/emc-mongoose/mongoose-storage-driver-fs) | [VFS](https://www.oreilly.com/library/view/understanding-the-linux/0596005652/ch12s01.html) storage driver, extends the NIO storage driver primitive | ![Maven metadata URL](https://img.shields.io/maven-metadata/v/http/central.maven.org/maven2/com/github/emc-mongoose/mongoose-storage-driver-fs/maven-metadata.xml.svg?label=%20) | ![Gitlab pipeline status](https://img.shields.io/gitlab/pipeline/emc-mongoose/mongoose-storage-driver-fs.svg?label=%20) | [FS](https://mongoose-issues.atlassian.net/projects/FS) 27 | | [mongoose-storage-driver-**atmos**](https://github.com/emc-mongoose/mongoose-storage-driver-atmos) | [Dell EMC Atmos](https://poland.emc.com/collateral/software/data-sheet/h5770-atmos-ds.pdf) storage driver, extends the HTTP storage driver primitive | ![Maven metadata URL](https://img.shields.io/maven-metadata/v/http/central.maven.org/maven2/com/github/emc-mongoose/mongoose-storage-driver-atmos/maven-metadata.xml.svg?label=%20) | ![Gitlab pipeline status](https://img.shields.io/gitlab/pipeline/emc-mongoose/mongoose-storage-driver-atmos.svg?label=%20) | [BASE](https://mongoose-issues.atlassian.net/projects/BASE) 28 | | [mongoose-storage-driver-**s3**](https://github.com/emc-mongoose/mongoose-storage-driver-s3) | [Amazon S3](https://docs.aws.amazon.com/en_us/AmazonS3/latest/API/Welcome.html) storage driver, extends the HTTP storage driver primitive | ![Maven metadata URL](https://img.shields.io/maven-metadata/v/http/central.maven.org/maven2/com/github/emc-mongoose/mongoose-storage-driver-s3/maven-metadata.xml.svg?label=%20) | ![Gitlab pipeline status](https://img.shields.io/gitlab/pipeline/emc-mongoose/mongoose-storage-driver-s3.svg?label=%20) | [S3](https://mongoose-issues.atlassian.net/projects/S3) 29 | | [mongoose-storage-driver-**swift**](https://github.com/emc-mongoose/mongoose-storage-driver-swift) | [OpenStack Swift](https://wiki.openstack.org/wiki/Swift) storage driver, extends the HTTP storage driver primitive | ![Maven metadata URL](https://img.shields.io/maven-metadata/v/http/central.maven.org/maven2/com/github/emc-mongoose/mongoose-storage-driver-swift/maven-metadata.xml.svg?label=%20) | ![Gitlab pipeline status](https://img.shields.io/gitlab/pipeline/emc-mongoose/mongoose-storage-driver-swift.svg?label=%20) | [SWIFT](https://mongoose-issues.atlassian.net/projects/SWIFT) 30 | 31 | # Additional Extensions 32 | 33 | The *additional extension* are not included in this bundle. 34 | 35 | | Repo | Description | Latest Release | Integration Status | Issue Tracker Link | 36 | |------|-------------|---------|-------------------------------|--------| 37 | | [mongoose-storage-driver-**preempt**](https://github.com/emc-mongoose/mongoose-storage-driver-preempt) | Preemptive multitasking storage driver primitive, using thread-per-task approach for the I/O | ![Maven metadata URL](https://img.shields.io/maven-metadata/v/http/central.maven.org/maven2/com/github/emc-mongoose/mongoose-storage-driver-preempt/maven-metadata.xml.svg?label=%20) | ![Gitlab pipeline status](https://img.shields.io/gitlab/pipeline/emc-mongoose/mongoose-storage-driver-preempt.svg?label=%20) | [BASE](https://mongoose-issues.atlassian.net/projects/BASE) 38 | | [mongoose-storage-driver-**hdfs**](https://github.com/emc-mongoose/mongoose-storage-driver-hdfs) | [Apache HDFS](http://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html) storage driver, extends the NIO storage driver primitive | ![Maven metadata URL](https://img.shields.io/maven-metadata/v/http/central.maven.org/maven2/com/github/emc-mongoose/mongoose-storage-driver-hdfs/maven-metadata.xml.svg?label=%20) | ![Gitlab pipeline status](https://img.shields.io/gitlab/pipeline/emc-mongoose/mongoose-storage-driver-hdfs.svg?label=%20) | [HDFS](https://mongoose-issues.atlassian.net/projects/HDFS) 39 | | [mongoose-storage-driver-**pravega**](https://github.com/emc-mongoose/mongoose-storage-driver-pravega) | [Pravega](http://pravega.io) storage driver, extends the preemptive multitasking storage driver primitive | ![Maven metadata URL](https://img.shields.io/maven-metadata/v/http/central.maven.org/maven2/com/github/emc-mongoose/mongoose-storage-driver-pravega/maven-metadata.xml.svg?label=%20) | ![Gitlab pipeline status](https://img.shields.io/gitlab/pipeline/emc-mongoose/mongoose-storage-driver-pravega.svg?label=%20) | [PRAVEGA](https://mongoose-issues.atlassian.net/projects/PRAVEGA) 40 | | [mongoose-storage-driver-**kafka**](https://github.com/emc-mongoose/mongoose-storage-driver-kafka) | [Apache Kafka](https://kafka.apache.org/) storage driver, extends the preemptive multitasking storage driver primitive | ![Maven metadata URL](https://img.shields.io/maven-metadata/v/http/central.maven.org/maven2/com/github/emc-mongoose/mongoose-storage-driver-kafka/maven-metadata.xml.svg?label=%20) | ![Gitlab pipeline status](https://img.shields.io/gitlab/pipeline/emc-mongoose/mongoose-storage-driver-kafka.svg?label=%20) | [KAFKA](https://mongoose-issues.atlassian.net/projects/KAFKA) 41 | | [mongoose-storage-driver-**pulsar**](https://github.com/emc-mongoose/mongoose-storage-driver-pulsar) | [Apache Pulsar](https://pulsar.apache.org/) storage driver, extends the cooperative multitasking storage driver primitive | ![Maven metadata URL](https://img.shields.io/maven-metadata/v/http/central.maven.org/maven2/com/github/emc-mongoose/mongoose-storage-driver-pulsar/maven-metadata.xml.svg?label=%20) | ![Gitlab pipeline status](https://img.shields.io/gitlab/pipeline/emc-mongoose/mongoose-storage-driver-pulsar.svg?label=%20) | [PULSAR](https://mongoose-issues.atlassian.net/projects/PULSAR) 42 | 43 | # Auxiliary Tools 44 | 45 | | Repo | Description | Latest Release | Integration Status | Issue Tracker Link | 46 | |------|-------------|---------|-------------------------------|--------| 47 | | [darzee](https://github.com/emc-mongoose/darzee) | Mongoose GUI web application | TBD | TBD | [GUI](https://mongoose-issues.atlassian.net/browse/GUI) 48 | | [mongoose-helm-charts](https://github.com/emc-mongoose/mongoose-helm-charts) | [Helm](https://helm.sh/) charts to easily deploy Mongoose in [K8s](https://kubernetes.io/) environment | TBD | TBD | [HELM](https://mongoose-issues.atlassian.net/projects/HELM/issues) 49 | | [e2e-latency-generator](https://github.com/emc-mongoose/e2e-latency-generator) | The tool consuming the Mongoose's operations trace output data and producing the raw end-to-end latency data and heatmap chart | - | - | - 50 | | [scenario-converter-3to4](https://github.com/emc-mongoose/scenario-converter-3to4) | This tool converts the Json-scenarios used in the Mongoose v3.* into new JavaScript-scenarios. | - | - | - 51 | 52 | # Backward Compatibility Notes 53 | 54 | * The extensions are not overriding the base default options when launched from the jar file. E.g. the default storage 55 | port is 7 and the default storage driver is "dummy-mock". Override the defaults explicitly or consider using the 56 | Docker image. 57 | 58 | * The base Mongoose version and this bundle version may differ. The base version is used to determine the logs output 59 | path. 60 | 61 | Example: 62 | ```bash 63 | java -jar mongoose-bundle-.jar \ 64 | --storage-driver-type=s3 \ 65 | --storage-net-node-port=9020 66 | ``` 67 | 68 | # Build bundle 69 | 70 | ```bash 71 | ./gradlew clean jar 72 | ls -l build/libs 73 | ``` 74 | 75 | # Deploy 76 | 77 | ## Bare Jar Download 78 | 79 | https://repo.maven.apache.org/maven2/com/github/emc-mongoose/mongoose-bundle/ 80 | 81 | ## Docker 82 | 83 | ```bash 84 | docker run ... emcmongoose/mongoose[:] ... 85 | ``` 86 | 87 | # Dependency 88 | 89 | The following dependency graph should be considered when running Mongoose w/o Docker and using some specific extension. 90 | For example, using the `mongoose-storage-driver-hdfs` extension will require to have the `mongoose-storage-driver-coop` 91 | and `mongoose-storage-driver-nio` extensions in the `~/.mongoose//ext` directory (with proper versions). 92 | 93 | ``` 94 | mongoose-base 95 | |___ mongoose-load-step-pipeline 96 | |___ mongoose-load-step-weighted 97 | |___ mongoose-storage-driver-coop 98 | | |___ mongoose-storage-driver-netty 99 | | | |___ mongoose-storage-driver-http 100 | | | |___ mongoose-storage-driver-atmos 101 | | | |___ mongoose-storage-driver-s3 102 | | | |___ mongoose-storage-driver-swift 103 | | |___ mongoose-storage-driver-nio 104 | | | |___ mongoose-storage-driver-fs 105 | | | |___ mongoose-storage-driver-hdfs 106 | | |___ mongoose-storage-driver-pulsar 107 | |___ mongoose-storage-driver-preempt 108 | |___ mongoose-storage-driver-kafka 109 | |___ mongoose-storage-driver-pravega 110 | ``` 111 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Become one of the contributors! We thrive to build a welcoming and open community for anyone who wants to use Mongoose 2 | or contribute to it. 3 | [Here](https://github.com/thecodeteam/codedellemc.github.io/wiki/How-to-contribute-to-%7Bcode%7D-and-add-your-project) 4 | we describe how to contribute to any of the {code} projects. 5 | 6 | Please, note that all contributors shall follow the Contributor Agreement guidelines 7 | [provided here](https://github.com/thecodeteam/codedellemc.github.io/wiki/Contributor-Agreement). 8 | 9 | # Contents 10 | 11 | 1. [Contributors](#1-contributors)
12 | 1.2. [Roles](#12-roles)
13 | 2. [Versions](#2-versions)
14 | 2.1. [Backward Compatibility](#21-backward-compatibility)
15 | 2.2. [Numbers](#22-numbers)
16 | 3. [Issues](#3-issues)
17 | 3.1. [States](#31-states)
18 | 3.2. [Defects Priority](#32-defects-priority)
19 | 3.3. [Specific Properties](#33-specific-properties)
20 | 4. [Continuous Integration](#7-continuous-integration)
21 | 4.1. [Build](#41-build)
22 | 4.2. [Testing](#42-testing)
23 | 4.2.1. [Unit Tests](#421-unit-tests)
24 | 4.2.2. [Integration Tests](#422-integration-tests)
25 | 4.2.3. [Legacy System Tests](#423-system-tests)
26 | 4.2.3.1. [Containerized Tests](#4231-containerized-tests)
27 | 4.2.4. [Endurance Tests](#424-endurance-tests)
28 | 4.2.5. [Robot Tests](#425-robot-tests)
29 | 4.3. [Releasing](#43-releasing)
30 | 5. [Code](#6-code)
31 | 5.1. [Style](#51-style)
32 | 5.2. [Exception Handling](#52-exception-handling)
33 | 5.3. [Performance](#53-performance)
34 | 35 | # 1. Contributors 36 | 37 | Alphabetically: 38 | 39 | * [Andrey Kurilov](https://github.com/akurilov) 40 | * Gennady Eremeev 41 | * [Ilya Kisliakovsky](https://github.com/kisliakovsky) 42 | * [Kirill Gusakov](https://github.com/gusakk) 43 | * Mikhail Danilov 44 | * [Mikhail Malygin](https://github.com/aphreet) 45 | * [Olga Zhavzharova](https://github.com/Zhavzharova) 46 | * [Veronika Kochugova](https://github.com/veronikaKochugova) 47 | 48 | # 1.2. Roles 49 | 50 | | Name | Responsibilities | Current Assignees 51 | |------|------------------|------------------ 52 | | User | Report the issues | Definitely unknown 53 | | Developer |
  • Development
  • Testing
  • Automation
  • Documentation
|
  • Veronika Kochugova
  • Andrey Kurilov
      • 54 | | Owner |
      • The next version scope definition
      • Roadmap definition
      • User interaction
      |
      • Andrey Kurilov
      • Veronika Kochugova
      55 | 56 | # 2. Versions 57 | 58 | ## 2.1. Backward Compatibility 59 | 60 | The following interfaces are mentioned as the subject of the backward compatibility: 61 | 1. Input (item list files, scenario files, configuration options) 62 | 2. Output files containing the metrics 63 | 3. API 64 | 65 | ## 2.2. Numbers 66 | 67 | Mongoose uses the [semantic versioning](http://semver.org/). This means that the ***X.Y.Z*** version notation is used: 68 | 69 | * ***X***
      70 | Major version number. Points to significant design and interfaces change. The *backward compatibility* is **not 71 | guaranteed**. 72 | * ***Y***
      73 | Minor version number. The *backward compatibility* is guaranteed. 74 | * ***Z***
      75 | Patch version number. Includes only the defect fixes. 76 | 77 | # 3. Issues 78 | 79 | Types: 80 | * Defect 81 | * Story 82 | * Task 83 | * Sub-task 84 | 85 | | Type | Description | 86 | |----------|-------------| 87 | | Defect | The defect/bug which **affects the released version** (the type "Task" should be used if a defect/bug affects the version which is not released yet) | 88 | | Story | High-level use case or a long-term activity aspect (testing, performance, etc) | 89 | | Task | A task which couldn't be included into any defect/story | 90 | | Sub-task | A task which could be included into a defect/story | 91 | 92 | Tracker link: https://mongoose-issues.atlassian.net/projects/GOOSE 93 | 94 | ## 3.1. States 95 | 96 | | State | Description | 97 | |-------------|-------------| 98 | | OPEN | All new issues should have this state. The issues are selected from the set of the *OPEN* issues for the proposal and review process. The task is updated w/ the corresponding comment but left in the *OPEN* state if it's considered incomplete/incorrect. Also incomplete/incorrect issue should be assigned back to the reporter. 99 | | PROPOSED | The issue is selected for the approval by the *owners*. 100 | | DEFERRED | Owners have approved the issue to be processed after the next major/minor (non-patch) version is released. 101 | | ACCEPTED | Owners approved the issue to be processed before the next major/minor (non-patch) version is released. 102 | | ESCALATED | Critical defect which interrupts all *DEFERRED*/*ACCEPTED* issues processing. Causes the new *patch* version release ASAP. 103 | | IN PROGRESS | The issue is in progress currently either initially done and the corresponding merge request to the `master` branch is created 104 | | RESOLVED | Issue is done and the corresponding changes are merged into the `master` branch 105 | | CLOSED | The new version is released containing the corresponding changes 106 | 107 | **Note**: 108 | > The corresponding impact probability/frequency is not taken into account in the process currently. For example, all 109 | > defects are assumed to be equally frequently occurring and affecting same users, regardless the particular 110 | > scenario/use case. This approach is used due to the lack of the sufficient statistical information about the Mongoose 111 | > usage. 112 | 113 | ## 3.2. Defects Priority 114 | 115 | | Priority | Conditions | Target state as a result of the review 116 | |--------------|------------|--------------------------------------- 117 | | Critical | No workaround available **and** any of the following:
      • Crash
      • Hang
      • Not functioning
      • Functioning incorrectly
      • Performance degradation
      | `ESCALATED` 118 | | Non-critical | Not *critical* **and** not *minor* | `ACCEPTED` (for the next minor/major version) 119 | | Minor | Any of the following:
      • Usability issue
      • Cosmetic
      | `OPEN` or `ACCEPTED` 120 | 121 | ## 3.3. Specific properties 122 | 123 | | Name | Applicable Issue Types | Who is responsible to specify | Notes 124 | |-----------------------|------------------------|--------------------------------|-------| 125 | | Affected version | Defect | Reporter: user/developer/owner | Only the *latest* version may be used for the defect reporting. The issue should be *rejected* if the reported version is not *latest*. 126 | | Branch | Defect, Task, Sub-task | Reviewer: developer/owner | 127 | | Description | Task, Sub-task | Reporter: user/developer/owner | 128 | | Expected behaviour | Defect | Reporter: user/developer/owner | The reference to the particular documentation part describing the expected behavior is preferable. 129 | | Fix version | Defect, Task, Sub-task | Reviewer: developer/owner | 130 | | Limitations | Story | Reviewer: developer/owner | 131 | | Observed behaviour | Defect | Reporter: user/developer/owner | Error message, errors.log output file, etc. 132 | | Pull request | Defect, Task, Sub-task | Reviewer: developer/owner | 133 | | Resolution commit | Defect, Task, Sub-task | Reviewer: developer/owner | 134 | | Root cause | Defect | Reviewer: developer/owner | 135 | | Start command/request | Defect | Reporter: user/developer/owner | Leave only the essential things to reproduce: try to check if possible if the bug is reproducible w/o distributed mode, different concurrency level, item data size, etc. 136 | | Scenario | Defect | Reporter: user/developer/owner | Don't clutter with large scenario files. Simplify the scenario leaving only the essential things. 137 | | Steps | Defect | Reporter: user/developer/owner | 138 | | Purpose | Story | Reporter: user/developer/owner | Which particular problem should be solved with Mongoose? The links to the related documents and literature are encouraged. 139 | | Requirements | Story | Reporter: user/developer/owner | Both functional and performance requirements are mandatory. Optionally the additional requirements/possible enhancements may be specified. 140 | 141 | # 4. Continuous Integration 142 | 143 | https://gitlab.com/emcmongoose/mongoose/pipelines 144 | for Windows: https://ci.appveyor.com/project/veronikaKochugova/mongoose 145 | 146 | ## 4.1. Build 147 | 148 | Mongoose may be distributed as a single jar. JDK 11+ is required to build. 149 | 150 | ```bash 151 | ./gradlew clean jar 152 | ``` 153 | 154 | The resulting jar file path is `./build/libs/mongoose-.jar`. 155 | 156 | ## 4.2. Testing 157 | 158 | ### 4.2.1. Unit Tests 159 | 160 | ```bash 161 | ./gradlew clean test 162 | ``` 163 | 164 | ### 4.2.2. Integration Tests 165 | 166 | ```bash 167 | ./gradlew clean integrationTest 168 | ``` 169 | 170 | ### 4.2.3. Legacy System Tests 171 | 172 | The system tests use the [JUnit parameterization](https://github.com/junit-team/junit4/wiki/Parameterized-tests). The parameter values are taken from the environment. The list of the system tests parameters below: 173 | 174 | | Parameter Name | Acceptable Values | Values Meaning 175 | |----------------|----------------------------------------|---------------- 176 | | STORAGE_TYPE | s3, atmos, fs, swift | (the same) 177 | | RUN_MODE | local, distributed | 1, 2 178 | | CONCURRENCY | unlimited, single, small, medium, high | 0, 1, 10, 100, 1000 179 | | ITEM_SIZE | empty, small, medium, large, huge | 0, 10KB, 1MB, 100MB, 10GB 180 | 181 | To run the system tests for the particular case use the commands like 182 | below: 183 | 184 | ```bash 185 | export MONGOOSE_IMAGE_VERSION=testing 186 | export STORAGE_TYPE=s3 187 | export RUN_MODE=distributed 188 | export CONCURRENCY=medium 189 | export ITEM_SIZE=small 190 | ./gradlew clean systemTest --tests com.emc.mongoose.system.CircularAppendTest 191 | ``` 192 | 193 | Note that some system tests will not run for some parameter values. The acceptable parameter values are declared explicitly in the `.travis.yml` file. 194 | 195 | #### 4.2.3.1. Containerized Tests 196 | 197 | Since v4.0.0 all system tests are containerized. To run a system test locally it's necessary to 198 | prepare testing Docker image manually: 199 | 200 | ```bash 201 | ./gradlew buildImage 202 | ``` 203 | 204 | Also it's necessary to supply the testing image version to the system 205 | test via environment variable `MONGOOSE_IMAGE_VERSION`: 206 | 207 | ```bash 208 | export MONGOOSE_IMAGE_VERSION=testing 209 | export STORAGE_TYPE=atmos 210 | export RUN_MODE=distributed 211 | export CONCURRENCY=medium 212 | export ITEM_SIZE=small 213 | ./gradlew clean systemTest --tests com.emc.mongoose.system.CreateNoLimitTest 214 | ``` 215 | 216 | ### 4.2.4. Endurance Tests 217 | 218 | ```bash 219 | export MONGOOSE_IMAGE_VERSION=testing 220 | export STORAGE_TYPE=swift 221 | export RUN_MODE=distributed 222 | export CONCURRENCY=unlimited 223 | export ITEM_SIZE=large 224 | ./gradlew clean enduranceTest --tests com.emc.mongoose.endurance.ParallelPipelineAndInfiniteLoopTest 225 | ``` 226 | 227 | ### 4.2.5. Robot Tests 228 | 229 | *Note*: 230 | > Currently the Robot tests are failing when being executed locally and passing when being executed on the [Gitlab CI](https://gitlab.com/emcmongoose/mongoose/pipelines) 231 | 232 | ```bash 233 | export SUITE= 234 | export TEST= 235 | ./gradlew robotest 236 | ``` 237 | 238 | Example: 239 | ```bash 240 | SUITE=api.storage TEST=s3 ./gradlew clean robotest 241 | ``` 242 | 243 | ## 4.3. Releasing 244 | 245 | 1. Ensure all tests are OK 246 | 2. Ensure the new version documentation is ready 247 | 3. Share the testing build with QE and repeat this step until the qualification is OK 248 | 4. Create/replace the corresponding VCS tags: 249 | ```bash 250 | git tag -d latest 251 | git push origin :refs/tags/latest 252 | git tag -a .. -m .. 253 | git tag -a latest -m .. 254 | git push --tags --force 255 | ``` 256 | 5. Create the corresponding version branch `release-v..` and set the version in the configuration to 257 | .. 258 | 6. Deploy docker images, set the `..` and `latest` tags also for the Docker images. 259 | 7. Upload the artifacts to the Central Maven repo: 260 | 1. 261 | ```bash 262 | ./gradlew clean uploadArchives 263 | ``` 264 | 2. Go to the https://oss.sonatype.org/#stagingRepositories find the corresponding repository, close and then release 265 | it. 266 | 8. Update the projects depending on the Mongoose's API (storage drivers, at least) 267 | 268 | # 5. Code 269 | 270 | ## 5.1. Style 271 | 272 | [Google Java Style](https://google.github.io/styleguide/javaguide.html) is used as default. 273 | 274 | ### Autoformatting hook 275 | 276 | Git autoformatting hook reformats Java source code to comply with [Google Java Style](https://google.github.io/styleguide/javaguide.html). 277 | 278 | The hook script is in the root of the repository named `pre-commit`.`pre-commit` **MUST be copied to the directory** `.git/hooks/` and you need to check that the script has the access permissions to execute: 279 | ```bash 280 | ls -l pre-commit 281 | ``` 282 | If not, assign permissions for execution: 283 | ```bash 284 | chmod +x pre-commit 285 | ``` 286 | This hook will work automatically with any commit and format the code in the same style. 287 | 288 | ### In addition: 289 | 290 | * If interface is named `Foo` then: 291 | * Abstract implementation should be named as `FooBase` 292 | * Default concrete implementation should be names as `FooImpl` 293 | * Any field/local variable should be *final* if possible 294 | 295 | ## 5.2. Exception Handling 296 | 297 | The threads are not used in the usual way (*fibers* are used instead for the multitasking purposes). Therefore, having 298 | an `InterruptedException` thrown means that the run was interrupted externally. To stop the run, it's necessary to pass 299 | the exception to the uppermost level of the call stack. However, the `InterruptedException` is a checked 300 | exception and usually couldn't be passed outward. The utility method 301 | `com.github.akurilov.commons.lang.Exceptions#throwUnchecked` should be used for this: 302 | 303 | ```java 304 | ... 305 | import static com.github.akurilov.commons.lang.Exceptions.throwUnchecked; 306 | ... 307 | try { 308 | foo(); // may throw an InterruptedException 309 | } catch(final InterruptedException e) { 310 | throwUnchecked(e); 311 | } 312 | ``` 313 | 314 | The following exceptions catching should be avoided as far as the `InterruptedException` may be swallowed occasionally: 315 | 1. `Throwable` 316 | 2. `Exception` 317 | 318 | ## 5.3. Performance 319 | Take care about the performance in the ***critical*** places: 320 | * Avoid *frequent* objects instantiation 321 | * Avoid unnecessary *frequent* allocation 322 | * Avoid *frequent* method calls if possible 323 | * Avoid deep call stack if possible 324 | * Avoid I/O threads blocking 325 | * Avoid anonymous classes in the time-critical code 326 | * Avoid non-static inner classes in the time-critical code 327 | * Use thread locals (encryption, string builders) 328 | * Use buffering, buffer everything 329 | * Use batch processing if possible 330 | --------------------------------------------------------------------------------