├── .github ├── CODEOWNERS ├── dependabot.yml └── workflows │ ├── cd.yaml │ └── jenkins-security-scan.yml ├── .gitignore ├── .mvn ├── extensions.xml └── maven.config ├── Jenkinsfile ├── README.md ├── images ├── bucket-settings.png ├── cannot-assume-role.png ├── cloud-provider-configured.png ├── cloud-provider-no-configured.png ├── configure-credentials.png ├── custom-s3-service-configuration.png ├── fsj-step-archive.png ├── no-valid-iam-role.png ├── s3-bucket-name-not-valid.png ├── save-wrong-configuration.png ├── unable-to-get-credentials-from-environment.png ├── unable-to-get-region-from-environment.png ├── validation-access-denied.png ├── validation-access-disabled.png ├── validation-empty-bucket-name.png ├── validation-no-valid-credentials.png ├── validation-s3-bucket-does-not-exists.png ├── validation-session-expired.png ├── validation-success.png └── validation-wrong-region.png ├── pom.xml ├── sample-scripts ├── README.md ├── big-file.groovy ├── small-files.groovy └── stash.groovy └── src ├── main ├── java │ └── io │ │ └── jenkins │ │ └── plugins │ │ └── artifact_manager_jclouds │ │ ├── BlobStoreProvider.java │ │ ├── BlobStoreProviderDescriptor.java │ │ ├── JCloudsArtifactManager.java │ │ ├── JCloudsArtifactManagerFactory.java │ │ ├── JCloudsVirtualFile.java │ │ ├── TikaUtil.java │ │ └── s3 │ │ ├── S3BlobStore.java │ │ └── S3BlobStoreConfig.java └── resources │ ├── index.jelly │ └── io │ └── jenkins │ └── plugins │ └── artifact_manager_jclouds │ ├── JCloudsArtifactManagerFactory │ └── config.jelly │ └── s3 │ ├── S3BlobStore │ ├── config.jelly │ └── config.properties │ └── S3BlobStoreConfig │ ├── config.jelly │ ├── help-container.html │ ├── help-customEndpoint.html │ ├── help-customSigningRegion.html │ ├── help-deleteArtifacts.html │ ├── help-deleteStashes.html │ ├── help-disableSessionToken.html │ ├── help-prefix.html │ ├── help-useHttp.html │ ├── help-usePathStyleUrl.html │ └── help-useTransferAcceleration.html └── test ├── java └── io │ └── jenkins │ └── plugins │ └── artifact_manager_jclouds │ ├── MockApiMetadata.java │ ├── MockApiMetadataTest.java │ ├── MockBlobStore.java │ ├── MockBlobStoreTest.java │ ├── NetworkTest.java │ └── s3 │ ├── AbstractIntegrationTest.java │ ├── ConfigAsCodeTest.java │ ├── CustomBehaviorBlobStoreProvider.java │ ├── JCloudsArtifactManagerTest.java │ ├── JCloudsVirtualFileTest.java │ ├── LocalStackIntegrationTest.java │ ├── MinioIntegrationTest.java │ ├── S3AbstractTest.java │ ├── S3BlobStoreConfigFipsEnabledTest.java │ └── S3BlobStoreConfigTest.java └── resources └── io └── jenkins └── plugins └── artifact_manager_jclouds └── s3 └── configuration-as-code.yml /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @jenkinsci/artifact-manager-s3-plugin-developers 2 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: maven 4 | directory: "/" 5 | schedule: 6 | interval: weekly 7 | open-pull-requests-limit: 10 8 | ignore: 9 | - dependency-name: com.google.inject.extensions:guice-assistedinject 10 | - package-ecosystem: github-actions 11 | directory: / 12 | schedule: 13 | interval: weekly 14 | -------------------------------------------------------------------------------- /.github/workflows/cd.yaml: -------------------------------------------------------------------------------- 1 | # Note: additional setup is required, see https://www.jenkins.io/redirect/continuous-delivery-of-plugins 2 | 3 | name: cd 4 | on: 5 | workflow_dispatch: 6 | check_run: 7 | types: 8 | - completed 9 | 10 | jobs: 11 | maven-cd: 12 | uses: jenkins-infra/github-reusable-workflows/.github/workflows/maven-cd.yml@v1 13 | secrets: 14 | MAVEN_USERNAME: ${{ secrets.MAVEN_USERNAME }} 15 | MAVEN_TOKEN: ${{ secrets.MAVEN_TOKEN }} 16 | -------------------------------------------------------------------------------- /.github/workflows/jenkins-security-scan.yml: -------------------------------------------------------------------------------- 1 | name: Jenkins Security Scan 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | types: [ opened, synchronize, reopened ] 9 | workflow_dispatch: 10 | 11 | permissions: 12 | security-events: write 13 | contents: read 14 | actions: read 15 | 16 | jobs: 17 | security-scan: 18 | uses: jenkins-infra/jenkins-security-scan/.github/workflows/jenkins-security-scan.yaml@v2 19 | with: 20 | java-cache: 'maven' # Optionally enable use of a build dependency cache. Specify 'maven' or 'gradle' as appropriate. 21 | # java-version: 21 # Optionally specify what version of Java to set up for the build, or remove to use a recent default. 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | target 3 | 4 | # mvn hpi:run 5 | work 6 | 7 | # IntelliJ IDEA project files 8 | *.iml 9 | *.iws 10 | *.ipr 11 | .idea 12 | 13 | # Eclipse project files 14 | .settings 15 | .classpath 16 | .project 17 | 18 | .factorypath 19 | .stignore 20 | .vscode/ 21 | -------------------------------------------------------------------------------- /.mvn/extensions.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | io.jenkins.tools.incrementals 4 | git-changelist-maven-extension 5 | 1.8 6 | 7 | 8 | -------------------------------------------------------------------------------- /.mvn/maven.config: -------------------------------------------------------------------------------- 1 | -Pconsume-incrementals 2 | -Pmight-produce-incrementals 3 | -Dchangelist.format=%d.v%s 4 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | /* 2 | See the documentation for more options: 3 | https://github.com/jenkins-infra/pipeline-library/ 4 | */ 5 | buildPlugin( 6 | forkCount: '1C', // run this number of tests in parallel for faster feedback. If the number terminates with a 'C', the value will be multiplied by the number of available CPU cores 7 | useContainerAgent: false, // AbstractIntegrationTest 8 | configurations: [ 9 | [platform: 'linux', jdk: 21], 10 | [platform: 'windows', jdk: 17], 11 | ]) 12 | -------------------------------------------------------------------------------- /images/bucket-settings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkinsci/artifact-manager-s3-plugin/e7c0bffccd209e31feff8a5ffde791c5758d09f3/images/bucket-settings.png -------------------------------------------------------------------------------- /images/cannot-assume-role.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkinsci/artifact-manager-s3-plugin/e7c0bffccd209e31feff8a5ffde791c5758d09f3/images/cannot-assume-role.png -------------------------------------------------------------------------------- /images/cloud-provider-configured.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkinsci/artifact-manager-s3-plugin/e7c0bffccd209e31feff8a5ffde791c5758d09f3/images/cloud-provider-configured.png -------------------------------------------------------------------------------- /images/cloud-provider-no-configured.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkinsci/artifact-manager-s3-plugin/e7c0bffccd209e31feff8a5ffde791c5758d09f3/images/cloud-provider-no-configured.png -------------------------------------------------------------------------------- /images/configure-credentials.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkinsci/artifact-manager-s3-plugin/e7c0bffccd209e31feff8a5ffde791c5758d09f3/images/configure-credentials.png -------------------------------------------------------------------------------- /images/custom-s3-service-configuration.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkinsci/artifact-manager-s3-plugin/e7c0bffccd209e31feff8a5ffde791c5758d09f3/images/custom-s3-service-configuration.png -------------------------------------------------------------------------------- /images/fsj-step-archive.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkinsci/artifact-manager-s3-plugin/e7c0bffccd209e31feff8a5ffde791c5758d09f3/images/fsj-step-archive.png -------------------------------------------------------------------------------- /images/no-valid-iam-role.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkinsci/artifact-manager-s3-plugin/e7c0bffccd209e31feff8a5ffde791c5758d09f3/images/no-valid-iam-role.png -------------------------------------------------------------------------------- /images/s3-bucket-name-not-valid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkinsci/artifact-manager-s3-plugin/e7c0bffccd209e31feff8a5ffde791c5758d09f3/images/s3-bucket-name-not-valid.png -------------------------------------------------------------------------------- /images/save-wrong-configuration.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkinsci/artifact-manager-s3-plugin/e7c0bffccd209e31feff8a5ffde791c5758d09f3/images/save-wrong-configuration.png -------------------------------------------------------------------------------- /images/unable-to-get-credentials-from-environment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkinsci/artifact-manager-s3-plugin/e7c0bffccd209e31feff8a5ffde791c5758d09f3/images/unable-to-get-credentials-from-environment.png -------------------------------------------------------------------------------- /images/unable-to-get-region-from-environment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkinsci/artifact-manager-s3-plugin/e7c0bffccd209e31feff8a5ffde791c5758d09f3/images/unable-to-get-region-from-environment.png -------------------------------------------------------------------------------- /images/validation-access-denied.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkinsci/artifact-manager-s3-plugin/e7c0bffccd209e31feff8a5ffde791c5758d09f3/images/validation-access-denied.png -------------------------------------------------------------------------------- /images/validation-access-disabled.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkinsci/artifact-manager-s3-plugin/e7c0bffccd209e31feff8a5ffde791c5758d09f3/images/validation-access-disabled.png -------------------------------------------------------------------------------- /images/validation-empty-bucket-name.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkinsci/artifact-manager-s3-plugin/e7c0bffccd209e31feff8a5ffde791c5758d09f3/images/validation-empty-bucket-name.png -------------------------------------------------------------------------------- /images/validation-no-valid-credentials.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkinsci/artifact-manager-s3-plugin/e7c0bffccd209e31feff8a5ffde791c5758d09f3/images/validation-no-valid-credentials.png -------------------------------------------------------------------------------- /images/validation-s3-bucket-does-not-exists.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkinsci/artifact-manager-s3-plugin/e7c0bffccd209e31feff8a5ffde791c5758d09f3/images/validation-s3-bucket-does-not-exists.png -------------------------------------------------------------------------------- /images/validation-session-expired.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkinsci/artifact-manager-s3-plugin/e7c0bffccd209e31feff8a5ffde791c5758d09f3/images/validation-session-expired.png -------------------------------------------------------------------------------- /images/validation-success.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkinsci/artifact-manager-s3-plugin/e7c0bffccd209e31feff8a5ffde791c5758d09f3/images/validation-success.png -------------------------------------------------------------------------------- /images/validation-wrong-region.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkinsci/artifact-manager-s3-plugin/e7c0bffccd209e31feff8a5ffde791c5758d09f3/images/validation-wrong-region.png -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | 6 | org.jenkins-ci.plugins 7 | plugin 8 | 5.17 9 | 10 | 11 | io.jenkins.plugins 12 | artifact-manager-s3 13 | ${changelist} 14 | hpi 15 | 16 | 17 | 999999-SNAPSHOT 18 | 19 | 2.479 20 | ${jenkins.baseline}.3 21 | true 22 | Max 23 | Low 24 | 25 | 26 | Artifact Manager on S3 plugin 27 | A Jenkins plugin to keep artifacts and Pipeline stashes in Amazon S3. 28 | https://github.com/jenkinsci/${project.artifactId}-plugin 29 | 30 | 31 | MIT License 32 | https://opensource.org/licenses/MIT 33 | 34 | 35 | 36 | 37 | scm:git:https://github.com/jenkinsci/${project.artifactId}-plugin.git 38 | scm:git:git@github.com:jenkinsci/${project.artifactId}-plugin.git 39 | https://github.com/jenkinsci/${project.artifactId}-plugin 40 | ${scmTag} 41 | 42 | 43 | 44 | 45 | io.jenkins.plugins 46 | aws-global-configuration 47 | 48 | 49 | io.jenkins.plugins.aws-java-sdk2 50 | aws-java-sdk2-s3 51 | 52 | 53 | io.jenkins.plugins 54 | gson-api 55 | 56 | 57 | io.jenkins.plugins 58 | jaxb 59 | 60 | 61 | org.apache.tika 62 | tika-core 63 | 3.1.0 64 | 65 | 66 | org.slf4j 67 | * 68 | 69 | 70 | commons-io 71 | commons-io 72 | 73 | 74 | 75 | 76 | org.apache.jclouds.provider 77 | aws-s3 78 | 2.6.0 79 | 80 | 81 | 82 | com.google.code.gson 83 | gson 84 | 85 | 86 | 87 | com.sun.xml.bind 88 | jaxb-impl 89 | 90 | 91 | com.google.errorprone 92 | error_prone_annotations 93 | 94 | 95 | 96 | 97 | org.jenkins-ci.plugins 98 | aws-credentials 99 | 100 | 101 | com.google.errorprone 102 | error_prone_annotations 103 | 104 | 105 | 106 | 107 | software.amazon.awssdk 108 | sso 109 | 110 | 2.31.26 111 | test 112 | 113 | 114 | org.jenkins-ci.plugins 115 | apache-httpcomponents-client-4-api 116 | 117 | 118 | org.mockito 119 | mockito-core 120 | test 121 | 122 | 123 | org.eclipse.jdt 124 | org.eclipse.jdt.annotation 125 | 2.3.0 126 | test 127 | 128 | 129 | org.jenkins-ci.plugins 130 | structs 131 | 132 | 133 | org.jenkins-ci.plugins.workflow 134 | workflow-api 135 | 136 | 137 | org.jenkins-ci.plugins.workflow 138 | workflow-api 139 | tests 140 | test 141 | 142 | 143 | org.jenkins-ci.plugins.workflow 144 | workflow-step-api 145 | tests 146 | test 147 | 148 | 149 | org.jenkins-ci.test 150 | docker-fixtures 151 | 200.v22a_e8766731c 152 | test 153 | 154 | 155 | com.fasterxml.jackson.core 156 | jackson-databind 157 | 158 | 159 | commons-io 160 | commons-io 161 | 162 | 163 | 164 | 165 | org.jenkins-ci.plugins 166 | ssh-slaves 167 | 168 | 169 | com.google.errorprone 170 | error_prone_annotations 171 | 172 | 173 | test 174 | 175 | 176 | org.jenkins-ci.plugins 177 | jdk-tool 178 | test 179 | 180 | 181 | org.jenkins-ci.plugins.workflow 182 | workflow-job 183 | test 184 | 185 | 186 | org.jenkins-ci.plugins.workflow 187 | workflow-cps 188 | test 189 | 190 | 191 | org.jenkins-ci.plugins.workflow 192 | workflow-durable-task-step 193 | test 194 | 195 | 196 | org.jenkins-ci.plugins.workflow 197 | workflow-basic-steps 198 | test 199 | 200 | 201 | org.kohsuke.metainf-services 202 | metainf-services 203 | test 204 | 205 | 206 | org.jenkins-ci.plugins 207 | git 208 | test 209 | 210 | 211 | org.jenkins-ci.plugins 212 | git 213 | tests 214 | test 215 | 216 | 217 | org.jenkins-ci.plugins.workflow 218 | workflow-multibranch 219 | test 220 | 221 | 222 | org.jenkins-ci.plugins.workflow 223 | workflow-multibranch 224 | tests 225 | test 226 | 227 | 228 | org.jenkins-ci.plugins 229 | scm-api 230 | test 231 | 232 | 233 | org.jenkins-ci.plugins 234 | scm-api 235 | tests 236 | test 237 | 238 | 239 | io.jenkins 240 | configuration-as-code 241 | test 242 | 243 | 244 | org.testcontainers 245 | testcontainers 246 | test 247 | 248 | 249 | org.apache.commons 250 | commons-compress 251 | 252 | 253 | org.slf4j 254 | * 255 | 256 | 257 | 258 | 259 | org.testcontainers 260 | localstack 261 | test 262 | 263 | 264 | org.testcontainers 265 | minio 266 | test 267 | 268 | 269 | 270 | org.apache.commons 271 | commons-compress 272 | 1.27.1 273 | test 274 | 275 | 276 | 277 | 278 | 279 | io.jenkins.tools.bom 280 | bom-${jenkins.baseline}.x 281 | 4710.v016f0a_07e34d 282 | import 283 | pom 284 | 285 | 286 | org.testcontainers 287 | testcontainers-bom 288 | 1.21.0 289 | import 290 | pom 291 | 292 | 293 | 294 | com.google.inject.extensions 295 | guice-assistedinject 296 | 6.0.0 297 | 298 | 299 | 300 | 301 | 302 | 303 | 304 | org.apache.maven.plugins 305 | maven-enforcer-plugin 306 | 307 | 308 | 309 | 310 | com.google.inject:guice 311 | com.google.inject.extensions:guice-assistedinject 312 | 313 | 314 | 315 | 316 | 317 | 318 | 319 | display-info 320 | 321 | 322 | 323 | 324 | com.google.inject:guice 325 | com.google.inject.extensions:guice-assistedinject 326 | 327 | 328 | 329 | 330 | 331 | 332 | 333 | 334 | 335 | 336 | 337 | 338 | repo.jenkins-ci.org 339 | https://repo.jenkins-ci.org/public/ 340 | 341 | 342 | 343 | 344 | repo.jenkins-ci.org 345 | https://repo.jenkins-ci.org/public/ 346 | 347 | 348 | 349 | 350 | 351 | S3-settings 352 | 353 | 354 | S3_BUCKET 355 | 356 | 357 | 358 | 359 | 360 | org.apache.maven.plugins 361 | maven-surefire-plugin 362 | 363 | 364 | ${AWS_PROFILE} 365 | ${AWS_REGION} 366 | ${S3_BUCKET} 367 | ${S3_DIR} 368 | 369 | 370 | 371 | 372 | 373 | 374 | 375 | 376 | -------------------------------------------------------------------------------- /sample-scripts/README.md: -------------------------------------------------------------------------------- 1 | Example Pipeline scripts exercising various aspects of artifact storage. 2 | Will generally need `S3_BUCKET` and `S3_DIR` variables defined somewhere. 3 | -------------------------------------------------------------------------------- /sample-scripts/big-file.groovy: -------------------------------------------------------------------------------- 1 | def name = "test-s3-big-file" 2 | def label = "${name}-${UUID.randomUUID().toString()}" 3 | def file = "CentOS-7-x86_64-DVD-1505-01.iso" 4 | def uri = "s3://${S3_BUCKET}/${S3_DIR}${file}" 5 | 6 | timestamps { 7 | podTemplate(name: name, label: label, 8 | containers: [ 9 | containerTemplate(name: 'aws', image: 'mesosphere/aws-cli', ttyEnabled: true, command: 'cat') 10 | ]) { 11 | 12 | node(label) { 13 | stage('CLI Download') { 14 | container('aws') { 15 | sh "aws s3 cp ${uri} ." 16 | } 17 | } 18 | sh "ls -laFh" 19 | stage('Archive') { 20 | archiveArtifacts file 21 | } 22 | sh "ls -laFh" 23 | stage('Unarchive') { 24 | unarchive mapping: ["${file}": 'CentOS-7-x86_64-DVD-1505-01-unarchived.iso'] 25 | } 26 | sh "ls -laFh" 27 | container('aws') { 28 | sh "aws s3 rm ${uri}-uploaded" 29 | } 30 | stage('CLI Upload') { 31 | container('aws') { 32 | sh "aws s3 cp ${file} ${uri}-uploaded" 33 | } 34 | } 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /sample-scripts/small-files.groovy: -------------------------------------------------------------------------------- 1 | def name = "test-s3-small-files" 2 | def label = "${name}-${UUID.randomUUID().toString()}" 3 | def uri = "s3://${S3_BUCKET}/${S3_DIR}small-files/" 4 | 5 | timestamps { 6 | podTemplate(name: name, label: label, 7 | containers: [ 8 | containerTemplate(name: 'aws', image: 'mesosphere/aws-cli', ttyEnabled: true, command: 'cat') 9 | ]) { 10 | 11 | node(label) { 12 | stage('Setup') { 13 | 1.upto(100) { 14 | writeFile file: "test/test-${it}.txt", text: "test ${it}" 15 | } 16 | } 17 | stage('Archive') { 18 | sh "ls -laFhR" 19 | archiveArtifacts "test/*" 20 | } 21 | stage('Unarchive') { 22 | dir('unarch') { 23 | deleteDir() 24 | unarchive mapping: ["test/": '.'] 25 | sh "ls -laFhR" 26 | } 27 | } 28 | stage('Upload') { 29 | container('aws') { 30 | // sh "aws s3 rm ${uri}-uploaded" 31 | sh "aws s3 sync test/ ${uri}" 32 | } 33 | } 34 | stage('Download') { 35 | container('aws') { 36 | sh "aws s3 sync ${uri} test/" 37 | } 38 | } 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /sample-scripts/stash.groovy: -------------------------------------------------------------------------------- 1 | def name = "test-s3-stash" 2 | def label = "${name}-${UUID.randomUUID().toString()}" 3 | 4 | timestamps { 5 | podTemplate(name: name, label: label, 6 | containers: [ 7 | containerTemplate(name: 'aws', image: 'mesosphere/aws-cli', ttyEnabled: true, command: 'cat') 8 | ]) { 9 | 10 | node(label) { 11 | writeFile file: 'dir/stuff.txt', text: 'hello' 12 | writeFile file: 'temp', text: BUILD_ID 13 | archiveArtifacts 'dir/' 14 | stash name: 'stuff', includes: 'temp' 15 | } 16 | node(label) { 17 | dir('unarch') { 18 | unarchive mapping: [temp: 'temp'] 19 | } 20 | dir('unst') { 21 | unstash 'stuff' 22 | } 23 | } 24 | 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/io/jenkins/plugins/artifact_manager_jclouds/BlobStoreProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2018 CloudBees, Inc. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package io.jenkins.plugins.artifact_manager_jclouds; 26 | 27 | import java.io.IOException; 28 | import java.io.Serializable; 29 | import java.net.URI; 30 | import java.net.URL; 31 | 32 | import org.jclouds.blobstore.BlobStoreContext; 33 | import org.jclouds.blobstore.domain.Blob; 34 | import org.kohsuke.accmod.Restricted; 35 | import org.kohsuke.accmod.restrictions.Beta; 36 | 37 | import edu.umd.cs.findbugs.annotations.NonNull; 38 | import hudson.ExtensionPoint; 39 | import hudson.model.AbstractDescribableImpl; 40 | 41 | /** 42 | * Provider for jclouds-based blob stores usable for artifact storage. 43 | * An instance will be copied into a build record together with any fields it defines. 44 | */ 45 | @Restricted(Beta.class) 46 | public abstract class BlobStoreProvider extends AbstractDescribableImpl implements ExtensionPoint, Serializable { 47 | 48 | private static final long serialVersionUID = -861350249543443493L; 49 | 50 | public enum HttpMethod { 51 | GET, PUT; 52 | } 53 | 54 | /** A constant for the blob path prefix to use. */ 55 | @NonNull 56 | public abstract String getPrefix(); 57 | 58 | /** A constant for the blob container name to use. */ 59 | @NonNull 60 | public abstract String getContainer(); 61 | 62 | /** A constant to define whether we should delete artifacts or leave them to be managed on the blob service side. */ 63 | public abstract boolean isDeleteArtifacts(); 64 | 65 | /** A constant to define whether we should delete stashes or leave them to be managed on the blob service side. */ 66 | public abstract boolean isDeleteStashes(); 67 | 68 | /** Creates the jclouds handle for working with blob. */ 69 | @NonNull 70 | public abstract BlobStoreContext getContext() throws IOException; 71 | 72 | /** 73 | * Get a provider-specific URI. 74 | * 75 | * @param container 76 | * container where this exists. 77 | * @param key 78 | * fully qualified name relative to the container. 79 | * @return the URI 80 | */ 81 | @NonNull 82 | public abstract URI toURI(@NonNull String container, @NonNull String key); 83 | 84 | /** 85 | * Generate a URL valid for downloading OR uploading the blob for a limited period of time 86 | * 87 | * @param blob 88 | * blob to generate the URL for 89 | * @param httpMethod 90 | * HTTP method to create a URL for (downloads or uploads) 91 | * @return the URL 92 | * @throws IOException 93 | */ 94 | @NonNull 95 | public abstract URL toExternalURL(@NonNull Blob blob, @NonNull HttpMethod httpMethod) throws IOException; 96 | 97 | @Override 98 | public BlobStoreProviderDescriptor getDescriptor() { 99 | return (BlobStoreProviderDescriptor) super.getDescriptor(); 100 | } 101 | 102 | } 103 | -------------------------------------------------------------------------------- /src/main/java/io/jenkins/plugins/artifact_manager_jclouds/BlobStoreProviderDescriptor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2018 CloudBees, Inc. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package io.jenkins.plugins.artifact_manager_jclouds; 26 | 27 | import hudson.model.Descriptor; 28 | import org.kohsuke.accmod.Restricted; 29 | import org.kohsuke.accmod.restrictions.Beta; 30 | 31 | /** 32 | * Descriptor type for {@link BlobStoreProvider}. 33 | */ 34 | @Restricted(Beta.class) 35 | public abstract class BlobStoreProviderDescriptor extends Descriptor {} 36 | -------------------------------------------------------------------------------- /src/main/java/io/jenkins/plugins/artifact_manager_jclouds/JCloudsArtifactManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2018 CloudBees, Inc. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package io.jenkins.plugins.artifact_manager_jclouds; 26 | 27 | import hudson.AbortException; 28 | import hudson.EnvVars; 29 | import hudson.FilePath; 30 | import hudson.Launcher; 31 | import hudson.Util; 32 | import hudson.model.BuildListener; 33 | import hudson.model.Run; 34 | import hudson.model.TaskListener; 35 | import hudson.remoting.VirtualChannel; 36 | import hudson.slaves.WorkspaceList; 37 | import hudson.util.DirScanner; 38 | import hudson.util.io.ArchiverFactory; 39 | import edu.umd.cs.findbugs.annotations.NonNull; 40 | import hudson.Functions; 41 | import io.jenkins.plugins.artifact_manager_jclouds.BlobStoreProvider.HttpMethod; 42 | import io.jenkins.plugins.httpclient.RobustHTTPClient; 43 | import java.io.File; 44 | import java.io.IOException; 45 | import java.io.InputStream; 46 | import java.io.OutputStream; 47 | import java.net.URI; 48 | import java.net.URL; 49 | import java.net.URLConnection; 50 | import java.nio.file.Files; 51 | import java.nio.file.InvalidPathException; 52 | import java.nio.file.Path; 53 | import java.nio.file.Paths; 54 | import java.util.ArrayList; 55 | import java.util.Collection; 56 | import java.util.HashMap; 57 | import java.util.Map; 58 | import java.util.logging.Level; 59 | import java.util.logging.Logger; 60 | import jenkins.MasterToSlaveFileCallable; 61 | import jenkins.model.ArtifactManager; 62 | import jenkins.util.VirtualFile; 63 | import org.apache.http.client.methods.HttpGet; 64 | import org.jclouds.blobstore.BlobStore; 65 | import org.jclouds.blobstore.BlobStoreContext; 66 | import org.jclouds.blobstore.BlobStores; 67 | import org.jclouds.blobstore.domain.Blob; 68 | import org.jclouds.blobstore.domain.StorageMetadata; 69 | import org.jclouds.blobstore.options.CopyOptions; 70 | import org.jclouds.blobstore.options.ListContainerOptions; 71 | import org.jenkinsci.plugins.workflow.flow.StashManager; 72 | import org.kohsuke.accmod.Restricted; 73 | import org.kohsuke.accmod.restrictions.NoExternalUse; 74 | 75 | import static io.jenkins.plugins.artifact_manager_jclouds.TikaUtil.detectByTika; 76 | 77 | /** 78 | * Jenkins artifact/stash implementation using any blob store supported by Apache jclouds. 79 | * To offer a new backend, implement {@link BlobStoreProvider}. 80 | */ 81 | @Restricted(NoExternalUse.class) 82 | public final class JCloudsArtifactManager extends ArtifactManager implements StashManager.StashAwareArtifactManager { 83 | 84 | private static final Logger LOGGER = Logger.getLogger(JCloudsArtifactManager.class.getName()); 85 | 86 | static RobustHTTPClient client = new RobustHTTPClient(); 87 | 88 | private final BlobStoreProvider provider; 89 | 90 | private transient String key; // e.g. myorg/myrepo/master/123 91 | 92 | JCloudsArtifactManager(@NonNull Run build, BlobStoreProvider provider) { 93 | this.provider = provider; 94 | onLoad(build); 95 | } 96 | 97 | private Object readResolve() { 98 | if (provider == null) { 99 | throw new IllegalStateException("Missing provider field"); 100 | } 101 | return this; 102 | } 103 | 104 | @Override 105 | public void onLoad(@NonNull Run build) { 106 | this.key = String.format("%s/%s", build.getParent().getFullName(), build.getNumber()); 107 | } 108 | 109 | private String getBlobPath(String path) { 110 | return getBlobPath(key, path); 111 | } 112 | 113 | private String getBlobPath(String key, String path) { 114 | return String.format("%s%s/%s", provider.getPrefix(), key, path); 115 | } 116 | 117 | /* 118 | * This could be called multiple times 119 | */ 120 | @Override 121 | public void archive(FilePath workspace, Launcher launcher, BuildListener listener, Map artifacts) 122 | throws IOException, InterruptedException { 123 | LOGGER.log(Level.FINE, "Archiving from {0}: {1}", new Object[] { workspace, artifacts }); 124 | Map contentTypes = workspace.act(new ContentTypeGuesser(new ArrayList<>(artifacts.values()), listener)); 125 | LOGGER.fine(() -> "guessing content types: " + contentTypes); 126 | Map artifactUrls = new HashMap<>(); 127 | BlobStore blobStore = getContext().getBlobStore(); 128 | 129 | // Map artifacts to urls for upload 130 | for (Map.Entry entry : artifacts.entrySet()) { 131 | String path = "artifacts/" + entry.getKey(); 132 | String blobPath = getBlobPath(path); 133 | Blob blob = blobStore.blobBuilder(blobPath).build(); 134 | blob.getMetadata().setContainer(provider.getContainer()); 135 | blob.getMetadata().getContentMetadata().setContentType(contentTypes.get(entry.getValue())); 136 | artifactUrls.put(entry.getValue(), provider.toExternalURL(blob, HttpMethod.PUT)); 137 | } 138 | 139 | workspace.act(new UploadToBlobStorage(artifactUrls, contentTypes, listener)); 140 | listener.getLogger().printf("Uploaded %s artifact(s) to %s%n", artifactUrls.size(), provider.toURI(provider.getContainer(), getBlobPath("artifacts/"))); 141 | } 142 | 143 | private static class ContentTypeGuesser extends MasterToSlaveFileCallable> { 144 | private static final long serialVersionUID = 1L; 145 | 146 | private final Collection relPaths; 147 | private final TaskListener listener; 148 | 149 | ContentTypeGuesser(Collection relPaths, TaskListener listener) { 150 | this.relPaths = relPaths; 151 | this.listener = listener; 152 | } 153 | 154 | @Override 155 | public Map invoke(File f, VirtualChannel channel) { 156 | Map contentTypes = new HashMap<>(); 157 | for (String relPath : relPaths) { 158 | File theFile = new File(f, relPath); 159 | try { 160 | String contentType = Files.probeContentType(theFile.toPath()); 161 | if (contentType == null) { 162 | contentType = URLConnection.guessContentTypeFromName(theFile.getName()); 163 | } 164 | if (contentType == null) { 165 | contentType = detectByTika(theFile); 166 | } 167 | contentTypes.put(relPath, contentType); 168 | } catch (IOException e) { 169 | Functions.printStackTrace(e, listener.error("Unable to determine content type for file: " + theFile)); 170 | // A content type must be specified; otherwise, the metadata signature will be computed from data that includes "Content-Type:", but no such HTTP header will be sent, and AWS will reject the request. 171 | contentTypes.put(relPath, "application/octet-stream"); 172 | } 173 | } 174 | return contentTypes; 175 | } 176 | } 177 | 178 | private static class UploadToBlobStorage extends MasterToSlaveFileCallable { 179 | private static final long serialVersionUID = 1L; 180 | 181 | private final Map artifactUrls; // e.g. "target/x.war", "http://..." 182 | private final Map contentTypes; // e.g. "target/x.zip, "application/zip" 183 | private final TaskListener listener; 184 | // Bind when constructed on the master side; on the agent side, deserialize the same configuration. 185 | private final RobustHTTPClient client = JCloudsArtifactManager.client; 186 | 187 | UploadToBlobStorage(Map artifactUrls, Map contentTypes, TaskListener listener) { 188 | this.artifactUrls = artifactUrls; 189 | this.contentTypes = contentTypes; 190 | this.listener = listener; 191 | } 192 | 193 | @Override 194 | public Void invoke(File f, VirtualChannel channel) throws IOException, InterruptedException { 195 | try { 196 | for (Map.Entry entry : artifactUrls.entrySet()) { 197 | client.uploadFile(new File(f, entry.getKey()), contentTypes.get(entry.getKey()), entry.getValue(), listener); 198 | } 199 | } finally { 200 | listener.getLogger().flush(); 201 | } 202 | return null; 203 | } 204 | } 205 | 206 | @Override 207 | public boolean delete() throws IOException, InterruptedException { 208 | String blobPath = getBlobPath(""); 209 | if (!provider.isDeleteArtifacts()) { 210 | LOGGER.log(Level.FINE, "Ignoring blob deletion: {0}", blobPath); 211 | return false; 212 | } 213 | return JCloudsVirtualFile.delete(provider, getContext().getBlobStore(), blobPath); 214 | } 215 | 216 | @Override 217 | public VirtualFile root() { 218 | return new JCloudsVirtualFile(provider, provider.getContainer(), getBlobPath("artifacts")); 219 | } 220 | 221 | @Override 222 | public void stash(String name, FilePath workspace, Launcher launcher, EnvVars env, TaskListener listener, String includes, String excludes, boolean useDefaultExcludes, boolean allowEmpty) throws IOException, InterruptedException { 223 | BlobStore blobStore = getContext().getBlobStore(); 224 | 225 | // Map stash to url for upload 226 | String path = getBlobPath("stashes/" + name + ".tgz"); 227 | Blob blob = blobStore.blobBuilder(path).build(); 228 | blob.getMetadata().setContainer(provider.getContainer()); 229 | // We don't care about content-type when stashing files 230 | blob.getMetadata().getContentMetadata().setContentType(null); 231 | URL url = provider.toExternalURL(blob, HttpMethod.PUT); 232 | FilePath tempDir = WorkspaceList.tempDir(workspace); 233 | if (tempDir == null) { 234 | throw new AbortException("Could not make temporary directory in " + workspace); 235 | } 236 | workspace.act(new Stash(url, provider.toURI(provider.getContainer(), path), includes, excludes, useDefaultExcludes, allowEmpty, tempDir.getRemote(), listener)); 237 | } 238 | 239 | private static final class Stash extends MasterToSlaveFileCallable { 240 | private static final long serialVersionUID = 1L; 241 | private final URL url; 242 | private final URI uri; 243 | private final String includes, excludes; 244 | private final boolean useDefaultExcludes; 245 | private final boolean allowEmpty; 246 | private final String tempDir; 247 | private final TaskListener listener; 248 | private final RobustHTTPClient client = JCloudsArtifactManager.client; 249 | 250 | Stash(URL url, URI uri, String includes, String excludes, boolean useDefaultExcludes, boolean allowEmpty, String tempDir, TaskListener listener) throws IOException { 251 | /** Actual destination as a presigned URL. */ 252 | this.url = url; 253 | /** Logical location for display purposes only. */ 254 | this.uri = uri; 255 | this.includes = includes; 256 | this.excludes = excludes; 257 | this.useDefaultExcludes = useDefaultExcludes; 258 | this.allowEmpty = allowEmpty; 259 | this.tempDir = tempDir; 260 | this.listener = listener; 261 | } 262 | 263 | @Override 264 | public Void invoke(File f, VirtualChannel channel) throws IOException, InterruptedException { 265 | // TODO use streaming upload rather than a temp file; is it necessary to set the content length in advance? 266 | // (we prefer not to upload individual files for stashes, so as to preserve symlinks & file permissions, as StashManager’s default does) 267 | Path tempDirP = Paths.get(tempDir); 268 | Files.createDirectories(tempDirP); 269 | Path tmp = Files.createTempFile(tempDirP, "stash", ".tgz"); 270 | try { 271 | int count; 272 | try (OutputStream os = Files.newOutputStream(tmp)) { 273 | count = new FilePath(f).archive(ArchiverFactory.TARGZ, os, new DirScanner.Glob(Util.fixEmpty(includes) == null ? "**" : includes, excludes, useDefaultExcludes)); 274 | } catch (InvalidPathException e) { 275 | throw new IOException(e); 276 | } 277 | if (count == 0 && !allowEmpty) { 278 | throw new AbortException("No files included in stash"); 279 | } 280 | client.uploadFile(tmp.toFile(), url, listener); 281 | listener.getLogger().printf("Stashed %d file(s) to %s%n", count, uri); 282 | return null; 283 | } finally { 284 | listener.getLogger().flush(); 285 | Files.delete(tmp); 286 | } 287 | } 288 | } 289 | 290 | @Override 291 | public void unstash(String name, FilePath workspace, Launcher launcher, EnvVars env, TaskListener listener) throws IOException, InterruptedException { 292 | BlobStore blobStore = getContext().getBlobStore(); 293 | 294 | // Map stash to url for download 295 | String blobPath = getBlobPath("stashes/" + name + ".tgz"); 296 | Blob blob = blobStore.getBlob(provider.getContainer(), blobPath); 297 | if (blob == null) { 298 | throw new AbortException( 299 | String.format("No such saved stash ‘%s’ found at %s/%s", name, provider.getContainer(), blobPath)); 300 | } 301 | URL url = provider.toExternalURL(blob, HttpMethod.GET); 302 | workspace.act(new Unstash(url, listener)); 303 | listener.getLogger().printf("Unstashed file(s) from %s%n", provider.toURI(provider.getContainer(), blobPath)); 304 | } 305 | 306 | private static final class Unstash extends MasterToSlaveFileCallable { 307 | private static final long serialVersionUID = 1L; 308 | private final URL url; 309 | private final TaskListener listener; 310 | private final RobustHTTPClient client = JCloudsArtifactManager.client; 311 | 312 | Unstash(URL url, TaskListener listener) throws IOException { 313 | this.url = url; 314 | this.listener = listener; 315 | } 316 | 317 | @Override 318 | public Void invoke(File f, VirtualChannel channel) throws IOException, InterruptedException { 319 | try { 320 | client.connect("download", "download " + RobustHTTPClient.sanitize(url) + " into " + f, c -> c.execute(new HttpGet(url.toString())), response -> { 321 | try (InputStream is = response.getEntity().getContent()) { 322 | new FilePath(f).untarFrom(is, FilePath.TarCompression.GZIP); 323 | // Note that this API currently offers no count of files in the tarball we could report. 324 | } 325 | }, listener); 326 | } finally { 327 | listener.getLogger().flush(); 328 | } 329 | return null; 330 | } 331 | } 332 | 333 | @Override 334 | public void clearAllStashes(TaskListener listener) throws IOException, InterruptedException { 335 | String stashPrefix = getBlobPath("stashes/"); 336 | 337 | if (!provider.isDeleteStashes()) { 338 | LOGGER.log(Level.FINE, "Ignoring stash deletion: {0}", stashPrefix); 339 | return; 340 | } 341 | 342 | BlobStore blobStore = getContext().getBlobStore(); 343 | int count = 0; 344 | try { 345 | for (StorageMetadata sm : BlobStores.listAll(blobStore, provider.getContainer(), ListContainerOptions.Builder.prefix(stashPrefix).recursive())) { 346 | String path = sm.getName(); 347 | assert path.startsWith(stashPrefix); 348 | LOGGER.fine("deleting " + path); 349 | blobStore.removeBlob(provider.getContainer(), path); 350 | count++; 351 | } 352 | } catch (RuntimeException x) { 353 | throw new IOException(x); 354 | } 355 | listener.getLogger().printf("Deleted %d stash(es) from %s%n", count, provider.toURI(provider.getContainer(), stashPrefix)); 356 | } 357 | 358 | @Override 359 | public void copyAllArtifactsAndStashes(Run to, TaskListener listener) throws IOException, InterruptedException { 360 | ArtifactManager am = to.pickArtifactManager(); 361 | if (!(am instanceof JCloudsArtifactManager)) { 362 | throw new AbortException("Cannot copy artifacts and stashes to " + to + " using " + am.getClass().getName()); 363 | } 364 | JCloudsArtifactManager dest = (JCloudsArtifactManager) am; 365 | String allPrefix = getBlobPath(""); 366 | BlobStore blobStore = getContext().getBlobStore(); 367 | int count = 0; 368 | try { 369 | for (StorageMetadata sm : BlobStores.listAll(blobStore, provider.getContainer(), ListContainerOptions.Builder.prefix(allPrefix).recursive())) { 370 | String path = sm.getName(); 371 | assert path.startsWith(allPrefix); 372 | String destPath = getBlobPath(dest.key, path.substring(allPrefix.length())); 373 | LOGGER.fine("copying " + path + " to " + destPath); 374 | blobStore.copyBlob(provider.getContainer(), path, provider.getContainer(), destPath, CopyOptions.NONE); 375 | count++; 376 | } 377 | } catch (RuntimeException x) { 378 | throw new IOException(x); 379 | } 380 | listener.getLogger().printf("Copied %d artifact(s)/stash(es) from %s to %s%n", count, provider.toURI(provider.getContainer(), allPrefix), provider.toURI(provider.getContainer(), dest.getBlobPath(""))); 381 | } 382 | 383 | private BlobStoreContext getContext() throws IOException { 384 | return provider.getContext(); 385 | } 386 | 387 | } 388 | -------------------------------------------------------------------------------- /src/main/java/io/jenkins/plugins/artifact_manager_jclouds/JCloudsArtifactManagerFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2018 CloudBees, Inc. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package io.jenkins.plugins.artifact_manager_jclouds; 26 | 27 | import hudson.Extension; 28 | import hudson.model.Run; 29 | import jenkins.model.ArtifactManager; 30 | import jenkins.model.ArtifactManagerFactory; 31 | import jenkins.model.ArtifactManagerFactoryDescriptor; 32 | import org.jenkinsci.Symbol; 33 | import org.kohsuke.accmod.Restricted; 34 | import org.kohsuke.accmod.restrictions.NoExternalUse; 35 | import org.kohsuke.stapler.DataBoundConstructor; 36 | 37 | /** 38 | * Factory for {@link ArtifactManager} 39 | */ 40 | @Restricted(NoExternalUse.class) 41 | public class JCloudsArtifactManagerFactory extends ArtifactManagerFactory { 42 | 43 | private final BlobStoreProvider provider; 44 | 45 | @DataBoundConstructor 46 | public JCloudsArtifactManagerFactory(BlobStoreProvider provider) { 47 | if (provider == null) { 48 | throw new IllegalArgumentException(); 49 | } 50 | this.provider = provider; 51 | } 52 | 53 | private Object readResolve() { 54 | if (provider == null) { 55 | throw new IllegalStateException("Missing provider field"); 56 | } 57 | return this; 58 | } 59 | 60 | public BlobStoreProvider getProvider() { 61 | return provider; 62 | } 63 | 64 | @Override 65 | public ArtifactManager managerFor(Run build) { 66 | return new JCloudsArtifactManager(build, provider); 67 | } 68 | 69 | @Symbol("jclouds") 70 | @Extension 71 | public static final class DescriptorImpl extends ArtifactManagerFactoryDescriptor { 72 | 73 | @Override 74 | public String getDisplayName() { 75 | return "Cloud Artifact Storage"; 76 | } 77 | 78 | } 79 | 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/io/jenkins/plugins/artifact_manager_jclouds/JCloudsVirtualFile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2018 CloudBees, Inc. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package io.jenkins.plugins.artifact_manager_jclouds; 26 | 27 | import static org.jclouds.blobstore.options.ListContainerOptions.Builder.*; 28 | 29 | import java.io.FileNotFoundException; 30 | import java.io.IOException; 31 | import java.io.InputStream; 32 | import java.net.URI; 33 | import java.net.URL; 34 | import java.util.ArrayDeque; 35 | import java.util.ArrayList; 36 | import java.util.Arrays; 37 | import java.util.Date; 38 | import java.util.Deque; 39 | import java.util.HashMap; 40 | import java.util.List; 41 | import java.util.Map; 42 | import java.util.logging.Level; 43 | import java.util.logging.Logger; 44 | import java.util.stream.StreamSupport; 45 | 46 | import org.jclouds.blobstore.BlobStore; 47 | import org.jclouds.blobstore.BlobStoreContext; 48 | import org.jclouds.blobstore.BlobStores; 49 | import org.jclouds.blobstore.domain.Blob; 50 | import org.jclouds.blobstore.domain.MutableBlobMetadata; 51 | import org.jclouds.blobstore.domain.StorageMetadata; 52 | import org.jclouds.blobstore.options.ListContainerOptions; 53 | import org.jclouds.rest.AuthorizationException; 54 | import org.kohsuke.accmod.Restricted; 55 | import org.kohsuke.accmod.restrictions.NoExternalUse; 56 | 57 | import edu.umd.cs.findbugs.annotations.CheckForNull; 58 | import edu.umd.cs.findbugs.annotations.NonNull; 59 | import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; 60 | 61 | import hudson.AbortException; 62 | import hudson.remoting.Callable; 63 | import io.jenkins.plugins.artifact_manager_jclouds.BlobStoreProvider.HttpMethod; 64 | import jenkins.util.VirtualFile; 65 | 66 | /** 67 | * JClouds BlobStore Guide 68 | */ 69 | @Restricted(NoExternalUse.class) 70 | public class JCloudsVirtualFile extends VirtualFile { 71 | 72 | private static final long serialVersionUID = -5126878907895121335L; 73 | 74 | private static final Logger LOGGER = Logger.getLogger(JCloudsVirtualFile.class.getName()); 75 | 76 | @NonNull 77 | private BlobStoreProvider provider; 78 | @NonNull 79 | private final String container; 80 | @NonNull 81 | private final String key; 82 | @CheckForNull 83 | private transient Blob blob; 84 | 85 | @SuppressFBWarnings(value = "SE_TRANSIENT_FIELD_NOT_RESTORED", 86 | justification = "This field is expected to be loaded by a provider instead of deserialization.") 87 | @CheckForNull 88 | private transient BlobStoreContext context; 89 | 90 | public JCloudsVirtualFile(@NonNull BlobStoreProvider provider, @NonNull String container, @NonNull String key) { 91 | this.provider = provider; 92 | this.container = container; 93 | this.key = key; 94 | assert !key.isEmpty(); 95 | assert !key.startsWith("/"); 96 | assert !key.endsWith("/"); 97 | } 98 | 99 | private JCloudsVirtualFile(@NonNull JCloudsVirtualFile related, @NonNull String key) { 100 | this(related.provider, related.container, key); 101 | context = related.context; 102 | } 103 | 104 | /** 105 | * Build jclouds blob context that is the base for all operations 106 | */ 107 | @Restricted(NoExternalUse.class) // testing only 108 | BlobStoreContext getContext() throws IOException { 109 | if (context == null) { 110 | context = provider.getContext(); 111 | } 112 | return context; 113 | } 114 | 115 | private String getContainer() { 116 | return container; 117 | } 118 | 119 | /** 120 | * Returns the full name, directories included 121 | */ 122 | private String getKey() { 123 | return key; 124 | } 125 | 126 | /** 127 | * Returns the base name 128 | */ 129 | @Override 130 | public String getName() { 131 | return key.replaceFirst(".+/", ""); 132 | } 133 | 134 | private Blob getBlob() throws IOException { 135 | if (blob == null) { 136 | LOGGER.log(Level.FINE, "checking for existence of blob {0} / {1}", new Object[] {container, key}); 137 | blob = getContext().getBlobStore().getBlob(getContainer(), getKey()); 138 | if (blob == null) { 139 | blob = getContext().getBlobStore().blobBuilder(getKey()).build(); 140 | blob.getMetadata().setContainer(getContainer()); 141 | } 142 | } 143 | return blob; 144 | } 145 | 146 | @Override 147 | public URI toURI() { 148 | return provider.toURI(container, key); 149 | } 150 | 151 | @Override 152 | public URL toExternalURL() throws IOException { 153 | return provider.toExternalURL(getBlob(), HttpMethod.GET); 154 | } 155 | 156 | @Override 157 | public VirtualFile getParent() { 158 | // undefined to go outside …/artifacts 159 | return new JCloudsVirtualFile(this, key.replaceFirst("/[^/]+$", "")); 160 | } 161 | 162 | @Override 163 | public boolean isDirectory() throws IOException { 164 | String keyS = key + "/"; 165 | CacheFrame frame = findCacheFrame(keyS); 166 | if (frame != null) { 167 | LOGGER.log(Level.FINER, "cache hit on directory status of {0} / {1}", new Object[] {container, key}); 168 | String relSlash = keyS.substring(frame.root.length()); // "" or "sub/dir/" 169 | return frame.children.keySet().stream().anyMatch(f -> f.startsWith(relSlash)); 170 | } 171 | LOGGER.log(Level.FINE, "checking directory status {0} / {1}", new Object[] {container, key}); 172 | return !getContext().getBlobStore().list(getContainer(), prefix(key + "/")).isEmpty(); 173 | } 174 | 175 | @Override 176 | public boolean isFile() throws IOException { 177 | CacheFrame frame = findCacheFrame(key); 178 | if (frame != null) { 179 | String rel = key.substring(frame.root.length()); 180 | CachedMetadata metadata = frame.children.get(rel); 181 | LOGGER.log(Level.FINER, "cache hit on file status of {0} / {1}", new Object[] {container, key}); 182 | return metadata != null; 183 | } 184 | LOGGER.log(Level.FINE, "checking file status {0} / {1}", new Object[] {container, key}); 185 | return getBlob().getMetadata().getSize() != null; 186 | } 187 | 188 | @Override 189 | public boolean exists() throws IOException { 190 | return isDirectory() || isFile(); 191 | } 192 | 193 | /** 194 | * List all the blobs under this one 195 | * 196 | * @return some blobs 197 | * @throws RuntimeException either now or when the stream is processed; wrap in {@link IOException} if desired 198 | */ 199 | private Iterable listStorageMetadata(boolean recursive) throws IOException { 200 | ListContainerOptions options = prefix(key + "/"); 201 | if (recursive) { 202 | options.recursive(); 203 | } 204 | return BlobStores.listAll(getContext().getBlobStore(), getContainer(), options); 205 | } 206 | 207 | @Override 208 | public VirtualFile[] list() throws IOException { 209 | String keyS = key + "/"; 210 | CacheFrame frame = findCacheFrame(keyS); 211 | if (frame != null) { 212 | LOGGER.log(Level.FINER, "cache hit on listing of {0} / {1}", new Object[] {container, key}); 213 | String relSlash = keyS.substring(frame.root.length()); // "" or "sub/dir/" 214 | return frame.children.keySet().stream(). // filenames relative to frame root 215 | filter(f -> f.startsWith(relSlash)). // those inside this dir 216 | map(f -> f.substring(relSlash.length()).replaceFirst("/.+", "")). // just the file simple name, or direct subdir name 217 | distinct(). // ignore duplicates if have multiple files under one direct subdir 218 | map(simple -> new JCloudsVirtualFile(this, keyS + simple)). // direct children 219 | toArray(VirtualFile[]::new); 220 | } 221 | VirtualFile[] list; 222 | try { 223 | list = StreamSupport.stream(listStorageMetadata(false).spliterator(), false) 224 | .map(meta -> new JCloudsVirtualFile(this, meta.getName().replaceFirst("/$", ""))) 225 | .toArray(VirtualFile[]::new); 226 | } catch (RuntimeException x) { 227 | throw new IOException(x); 228 | } 229 | LOGGER.log(Level.FINEST, "Listing files from {0} {1}: {2}", 230 | new String[] { getContainer(), getKey(), Arrays.toString(list) }); 231 | return list; 232 | } 233 | 234 | @Override 235 | public VirtualFile child(String name) { 236 | return new JCloudsVirtualFile(this, key + "/" + name); 237 | } 238 | 239 | @Override 240 | public long length() throws IOException { 241 | CacheFrame frame = findCacheFrame(key); 242 | if (frame != null) { 243 | String rel = key.substring(frame.root.length()); 244 | CachedMetadata metadata = frame.children.get(rel); 245 | LOGGER.log(Level.FINER, "cache hit on length of {0} / {1}", new Object[] {container, key}); 246 | return metadata != null ? metadata.length : 0; 247 | } 248 | LOGGER.log(Level.FINE, "checking length {0} / {1}", new Object[] {container, key}); 249 | MutableBlobMetadata metadata = getBlob().getMetadata(); 250 | Long size = metadata == null ? Long.valueOf(0) : metadata.getSize(); 251 | return size == null ? 0 : size; 252 | } 253 | 254 | @Override 255 | public long lastModified() throws IOException { 256 | CacheFrame frame = findCacheFrame(key); 257 | if (frame != null) { 258 | String rel = key.substring(frame.root.length()); 259 | CachedMetadata metadata = frame.children.get(rel); 260 | LOGGER.log(Level.FINER, "cache hit on lastModified of {0} / {1}", new Object[] {container, key}); 261 | return metadata != null ? metadata.lastModified : 0; 262 | } 263 | LOGGER.log(Level.FINE, "checking modification time {0} / {1}", new Object[] {container, key}); 264 | MutableBlobMetadata metadata = getBlob().getMetadata(); 265 | return metadata == null || metadata.getLastModified() == null ? 0 : metadata.getLastModified().getTime(); 266 | } 267 | 268 | @Override 269 | public boolean canRead() throws IOException { 270 | return true; 271 | } 272 | 273 | @Override 274 | public InputStream open() throws IOException { 275 | LOGGER.log(Level.FINE, "reading {0} / {1}", new Object[] {container, key}); 276 | if (isDirectory()) { 277 | // That is what java.io.FileInputStream.open throws 278 | throw new FileNotFoundException(String.format("%s/%s (Is a directory)", getContainer(), getKey())); 279 | } 280 | if (!isFile()) { 281 | throw new FileNotFoundException( 282 | String.format("%s/%s (No such file or directory)", getContainer(), getKey())); 283 | } 284 | return getBlob().getPayload().openStream(); 285 | } 286 | 287 | /** 288 | * Cache of metadata collected during {@link #run}. 289 | * Keys are {@link #container}. 290 | * Values are a stack of cache frames, one per nested {@link #run} call. 291 | */ 292 | private static final ThreadLocal>> cache = ThreadLocal.withInitial(HashMap::new); 293 | 294 | private static final class CacheFrame { 295 | /** {@link #key} of the root virtual file plus a trailing {@code /} */ 296 | final String root; 297 | /** 298 | * Information about all known (recursive) child files (not directories). 299 | * Keys are {@code /}-separated relative paths. 300 | * If the root itself happened to be a file, that information is not cached. 301 | */ 302 | final Map children; 303 | CacheFrame(String root, Map children) { 304 | this.root = root; 305 | this.children = children; 306 | } 307 | } 308 | 309 | /** 310 | * Record that a given file exists. 311 | */ 312 | private static final class CachedMetadata { 313 | final long length, lastModified; 314 | CachedMetadata(long length, long lastModified) { 315 | this.length = length; 316 | this.lastModified = lastModified; 317 | } 318 | } 319 | 320 | @Override 321 | public V run(Callable callable) throws IOException { 322 | LOGGER.log(Level.FINE, "enter cache {0} / {1}", new Object[] {container, key}); 323 | Deque stack = cacheFrames(); 324 | Map saved = new HashMap<>(); 325 | int prefixLength = key.length() + /* / */1; 326 | try { 327 | for (StorageMetadata sm : listStorageMetadata(true)) { 328 | Long length = sm.getSize(); 329 | if (length != null) { 330 | Date lastModified = sm.getLastModified(); 331 | saved.put(sm.getName().substring(prefixLength), new CachedMetadata(length, lastModified != null ? lastModified.getTime() : 0)); 332 | } 333 | } 334 | } catch (AuthorizationException e) { 335 | String cause = e.getCause() != null ? e.getCause().getMessage() : ""; 336 | throw new AbortException(String.format("Authorization failed: %s %s", e.getMessage(), cause)); 337 | } catch (RuntimeException x) { 338 | throw new IOException(x); 339 | } 340 | stack.push(new CacheFrame(key + "/", saved)); 341 | try { 342 | LOGGER.log(Level.FINE, "using cache {0} / {1}: {2} file entries", new Object[] {container, key, saved.size()}); 343 | return callable.call(); 344 | } finally { 345 | LOGGER.log(Level.FINE, "exit cache {0} / {1}", new Object[] {container, key}); 346 | stack.pop(); 347 | } 348 | } 349 | 350 | private Deque cacheFrames() { 351 | return cache.get().computeIfAbsent(container, c -> new ArrayDeque<>()); 352 | } 353 | 354 | /** Finds a cache frame whose {@link CacheFrame#root} is a prefix of the given {@link #key} or {@code /}-appended variant. */ 355 | private @CheckForNull CacheFrame findCacheFrame(String key) { 356 | return cacheFrames().stream().filter(frame -> key.startsWith(frame.root)).findFirst().orElse(null); 357 | } 358 | 359 | /** 360 | * Delete all blobs starting with a given prefix. 361 | */ 362 | public static boolean delete(BlobStoreProvider provider, BlobStore blobStore, String prefix) throws IOException, InterruptedException { 363 | try { 364 | List paths = new ArrayList<>(); 365 | for (StorageMetadata sm : BlobStores.listAll(blobStore, provider.getContainer(), ListContainerOptions.Builder.prefix(prefix).recursive())) { 366 | String path = sm.getName(); 367 | if (!path.startsWith(prefix)) { 368 | LOGGER.warning(() -> path + " does not start with " + prefix); 369 | continue; 370 | } 371 | paths.add(path); 372 | } 373 | if (paths.isEmpty()) { 374 | LOGGER.log(Level.FINE, "nothing to delete under {0}", prefix); 375 | return false; 376 | } else { 377 | LOGGER.log(Level.FINE, "deleting {0} blobs under {1}", new Object[] {paths.size(), prefix}); 378 | blobStore.removeBlobs(provider.getContainer(), paths); 379 | return true; 380 | } 381 | } catch (RuntimeException x) { 382 | throw new IOException(x); 383 | } 384 | } 385 | 386 | } 387 | -------------------------------------------------------------------------------- /src/main/java/io/jenkins/plugins/artifact_manager_jclouds/TikaUtil.java: -------------------------------------------------------------------------------- 1 | package io.jenkins.plugins.artifact_manager_jclouds; 2 | 3 | import org.apache.tika.Tika; 4 | 5 | import java.io.File; 6 | import java.io.IOException; 7 | 8 | public class TikaUtil { 9 | 10 | private static Tika tika = new Tika(); 11 | 12 | static String detectByTika(File f) throws IOException { 13 | return tika.detect(f); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/io/jenkins/plugins/artifact_manager_jclouds/s3/S3BlobStore.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2018 CloudBees, Inc. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package io.jenkins.plugins.artifact_manager_jclouds.s3; 26 | 27 | import java.io.IOException; 28 | import java.net.URI; 29 | import java.net.URISyntaxException; 30 | import java.net.URL; 31 | import java.time.Duration; 32 | import java.util.NoSuchElementException; 33 | import java.util.Properties; 34 | import java.util.logging.Level; 35 | import java.util.logging.Logger; 36 | 37 | import edu.umd.cs.findbugs.annotations.NonNull; 38 | 39 | import jenkins.security.FIPS140; 40 | import org.apache.commons.lang.StringUtils; 41 | import org.jclouds.ContextBuilder; 42 | import org.jclouds.aws.domain.SessionCredentials; 43 | import org.jclouds.aws.s3.AWSS3ProviderMetadata; 44 | import org.jclouds.blobstore.BlobStoreContext; 45 | import org.jclouds.blobstore.domain.Blob; 46 | import org.jclouds.domain.Credentials; 47 | import org.jclouds.location.reference.LocationConstants; 48 | import org.jclouds.osgi.ProviderRegistry; 49 | import org.jclouds.s3.reference.S3Constants; 50 | import org.kohsuke.accmod.Restricted; 51 | import org.kohsuke.accmod.restrictions.NoExternalUse; 52 | import org.kohsuke.stapler.DataBoundConstructor; 53 | 54 | import com.cloudbees.jenkins.plugins.awscredentials.AmazonWebServicesCredentials; 55 | import com.google.common.base.Supplier; 56 | 57 | import hudson.Extension; 58 | import io.jenkins.plugins.artifact_manager_jclouds.BlobStoreProvider; 59 | import io.jenkins.plugins.artifact_manager_jclouds.BlobStoreProviderDescriptor; 60 | import io.jenkins.plugins.aws.global_configuration.CredentialsAwsGlobalConfiguration; 61 | import org.jenkinsci.Symbol; 62 | import software.amazon.awssdk.auth.credentials.AwsCredentials; 63 | import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; 64 | import software.amazon.awssdk.regions.Region; 65 | import software.amazon.awssdk.services.s3.S3Client; 66 | import software.amazon.awssdk.services.s3.S3Configuration; 67 | import software.amazon.awssdk.services.s3.model.GetObjectRequest; 68 | import software.amazon.awssdk.services.s3.model.GetUrlRequest; 69 | import software.amazon.awssdk.services.s3.model.PutObjectRequest; 70 | import software.amazon.awssdk.services.s3.presigner.S3Presigner; 71 | import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; 72 | import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest; 73 | 74 | /** 75 | * Extension that customizes JCloudsBlobStore for AWS S3. Credentials are fetched from the environment, env vars, aws 76 | * profiles,... 77 | */ 78 | @Restricted(NoExternalUse.class) 79 | public class S3BlobStore extends BlobStoreProvider { 80 | 81 | private static final Logger LOGGER = Logger.getLogger(S3BlobStore.class.getName()); 82 | 83 | private static final long serialVersionUID = -8864075675579867370L; 84 | 85 | @DataBoundConstructor 86 | public S3BlobStore() { 87 | } 88 | 89 | @Override 90 | public String getPrefix() { 91 | return getConfiguration().getPrefix(); 92 | } 93 | 94 | @Override 95 | public String getContainer() { 96 | return getConfiguration().getContainer(); 97 | } 98 | 99 | public String getRegion() { 100 | return CredentialsAwsGlobalConfiguration.get().getRegion(); 101 | } 102 | 103 | public S3BlobStoreConfig getConfiguration(){ 104 | return S3BlobStoreConfig.get(); 105 | } 106 | 107 | @Override 108 | public boolean isDeleteArtifacts() { 109 | return getConfiguration().isDeleteArtifacts(); 110 | } 111 | 112 | @Override 113 | public boolean isDeleteStashes() { 114 | return getConfiguration().isDeleteStashes(); 115 | } 116 | 117 | @Override 118 | public BlobStoreContext getContext() throws IOException { 119 | LOGGER.log(Level.FINEST, "Building context"); 120 | ProviderRegistry.registerProvider(AWSS3ProviderMetadata.builder().build()); 121 | try { 122 | Properties props = new Properties(); 123 | boolean hasCustomEndpoint = StringUtils.isNotBlank(getConfiguration().getResolvedCustomEndpoint()); 124 | 125 | if(StringUtils.isNotBlank(getRegion())) { 126 | props.setProperty(LocationConstants.PROPERTY_REGIONS, getRegion()); 127 | } 128 | if (hasCustomEndpoint) { 129 | // We need to set the endpoint here and in the builder or listing 130 | // will still use s3.amazonaws.com 131 | props.setProperty(LocationConstants.ENDPOINT, getConfiguration().getResolvedCustomEndpoint()); 132 | } 133 | props.setProperty(S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS, Boolean.toString(!getConfiguration().getUsePathStyleUrl())); 134 | 135 | ContextBuilder builder = ContextBuilder.newBuilder("aws-s3") 136 | .credentialsSupplier(getCredentialsSupplier()) 137 | .overrides(props); 138 | 139 | if (hasCustomEndpoint) { 140 | builder = builder.endpoint(getConfiguration().getResolvedCustomEndpoint()); 141 | } 142 | 143 | return builder.buildView(BlobStoreContext.class); 144 | } catch (NoSuchElementException x) { 145 | throw new IOException(x); 146 | } 147 | } 148 | 149 | /** 150 | * field only for tests. 151 | */ 152 | static boolean BREAK_CREDS; 153 | 154 | /** 155 | * 156 | * @return the proper credential supplier using the configuration settings. 157 | * @throws IOException in case of error. 158 | */ 159 | private Supplier getCredentialsSupplier() throws IOException { 160 | // get user credentials from env vars, profiles,... 161 | String accessKeyId; 162 | String secretKey; 163 | String sessionToken; 164 | if (getConfiguration().getDisableSessionToken()) { 165 | AmazonWebServicesCredentials amazonWebServicesCredentials = CredentialsAwsGlobalConfiguration.get().getCredentials(); 166 | if (amazonWebServicesCredentials == null) { 167 | throw new IOException("No static AWS credentials found"); 168 | } 169 | AwsCredentials awsCredentials = amazonWebServicesCredentials.resolveCredentials(); 170 | accessKeyId = awsCredentials.accessKeyId(); 171 | secretKey = awsCredentials.secretAccessKey(); 172 | sessionToken = ""; 173 | } else { 174 | AwsSessionCredentials awsSessionCredentials = CredentialsAwsGlobalConfiguration.get() 175 | .sessionCredentials(getRegion(), CredentialsAwsGlobalConfiguration.get().getCredentialsId()); 176 | if(awsSessionCredentials != null ) { 177 | accessKeyId = awsSessionCredentials.accessKeyId(); 178 | secretKey = awsSessionCredentials.secretAccessKey(); 179 | sessionToken = awsSessionCredentials.sessionToken(); 180 | } else { 181 | throw new IOException("No session AWS credentials found"); 182 | } 183 | } 184 | 185 | if (BREAK_CREDS) { 186 | sessionToken = ""; 187 | } 188 | 189 | SessionCredentials sessionCredentials = SessionCredentials.builder() 190 | .accessKeyId(accessKeyId) 191 | .secretAccessKey(secretKey) 192 | .sessionToken(sessionToken) 193 | .build(); 194 | 195 | return () -> sessionCredentials; 196 | } 197 | 198 | @NonNull 199 | @Override 200 | public URI toURI(@NonNull String container, @NonNull String key) { 201 | try (S3Client s3Client = getConfiguration().getAmazonS3ClientBuilder().build()) { 202 | GetUrlRequest getUrlRequest = GetUrlRequest.builder().key(key).bucket(container).build(); 203 | URI uri = s3Client.utilities().getUrl(getUrlRequest).toURI(); 204 | LOGGER.fine(() -> container + " / " + key + " → " + uri); 205 | return uri; 206 | } catch (URISyntaxException e) { 207 | throw new IllegalStateException(e); 208 | } 209 | } 210 | 211 | /** 212 | * @see Generate a 213 | * Pre-signed Object URL using AWS SDK for Java 214 | */ 215 | @Override 216 | public URL toExternalURL(@NonNull Blob blob, @NonNull HttpMethod httpMethod) throws IOException { 217 | 218 | String customEndpoint = getConfiguration().getResolvedCustomEndpoint(); 219 | try (S3Client s3Client = getConfiguration().getAmazonS3ClientBuilderWithCredentials().build()) { 220 | S3Presigner.Builder presignerBuilder = S3Presigner.builder() 221 | .fipsEnabled(FIPS140.useCompliantAlgorithms()) 222 | .credentialsProvider(CredentialsAwsGlobalConfiguration.get().getCredentials()) 223 | .s3Client(s3Client); 224 | if (StringUtils.isNotBlank(customEndpoint)) { 225 | presignerBuilder.endpointOverride(URI.create(customEndpoint)); 226 | } 227 | 228 | String customRegion = getConfiguration().getCustomSigningRegion(); 229 | if(StringUtils.isBlank(customRegion)) { 230 | customRegion = CredentialsAwsGlobalConfiguration.get().getRegion(); 231 | } 232 | if(StringUtils.isNotBlank(customRegion)) { 233 | presignerBuilder.region(Region.of(customRegion)); 234 | } 235 | 236 | S3Configuration s3Configuration = S3Configuration.builder() 237 | .pathStyleAccessEnabled(getConfiguration().getUsePathStyleUrl()) 238 | .accelerateModeEnabled(getConfiguration().getUseTransferAcceleration()) 239 | .build(); 240 | presignerBuilder.serviceConfiguration(s3Configuration); 241 | 242 | try (S3Presigner presigner = presignerBuilder.build()) { 243 | Duration expiration = Duration.ofHours(1); 244 | String container = blob.getMetadata().getContainer(); 245 | String name = blob.getMetadata().getName(); 246 | LOGGER.log(Level.FINE, "Generating presigned URL for {0} / {1} for method {2}", 247 | new Object[]{container, name, httpMethod}); 248 | String contentType; 249 | switch (httpMethod) { 250 | case PUT: 251 | // Only set content type for upload URLs, so that the right S3 metadata gets set 252 | contentType = blob.getMetadata().getContentMetadata().getContentType(); 253 | PutObjectRequest putObjectRequest = PutObjectRequest.builder().bucket(container) 254 | .contentType(contentType) 255 | .key(name) 256 | .build(); 257 | PutObjectPresignRequest putObjectPresignRequest = PutObjectPresignRequest.builder() 258 | .signatureDuration(expiration) 259 | .putObjectRequest(putObjectRequest).build(); 260 | return presigner.presignPutObject(putObjectPresignRequest).url(); 261 | case GET: 262 | GetObjectRequest getObjectRequest = GetObjectRequest.builder().bucket(container).key(name).build(); 263 | GetObjectPresignRequest getObjectPresignRequest = GetObjectPresignRequest.builder() 264 | .signatureDuration(expiration) 265 | .getObjectRequest(getObjectRequest).build(); 266 | return presigner.presignGetObject(getObjectPresignRequest).url(); 267 | default: 268 | throw new IOException("HTTP Method " + httpMethod + " not supported for S3"); 269 | } 270 | 271 | } 272 | } 273 | } 274 | 275 | @Symbol("s3") 276 | @Extension 277 | public static final class DescriptorImpl extends BlobStoreProviderDescriptor { 278 | 279 | @Override 280 | public String getDisplayName() { 281 | return "Amazon S3"; 282 | } 283 | 284 | /** 285 | * 286 | * @return true if a container is configured. 287 | */ 288 | public boolean isConfigured(){ 289 | return StringUtils.isNotBlank(S3BlobStoreConfig.get().getContainer()); 290 | } 291 | 292 | } 293 | 294 | @Override 295 | public String toString() { 296 | final StringBuilder sb = new StringBuilder("S3BlobStore{"); 297 | sb.append("container='").append(getContainer()).append('\''); 298 | sb.append(", prefix='").append(getPrefix()).append('\''); 299 | sb.append(", region='").append(getRegion()).append('\''); 300 | sb.append(", deleteArtifacts='").append(isDeleteArtifacts()).append('\''); 301 | sb.append(", deleteStashes='").append(isDeleteStashes()).append('\''); 302 | sb.append('}'); 303 | return sb.toString(); 304 | } 305 | 306 | } 307 | -------------------------------------------------------------------------------- /src/main/java/io/jenkins/plugins/artifact_manager_jclouds/s3/S3BlobStoreConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2018 CloudBees, Inc. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package io.jenkins.plugins.artifact_manager_jclouds.s3; 26 | 27 | import java.io.IOException; 28 | import java.net.URI; 29 | import java.net.URISyntaxException; 30 | import java.util.regex.Pattern; 31 | 32 | import org.apache.commons.lang.StringUtils; 33 | import org.kohsuke.stapler.DataBoundSetter; 34 | import org.kohsuke.stapler.QueryParameter; 35 | import org.kohsuke.stapler.interceptor.RequirePOST; 36 | 37 | import com.google.common.annotations.VisibleForTesting; 38 | 39 | import edu.umd.cs.findbugs.annotations.NonNull; 40 | import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; 41 | 42 | import hudson.Extension; 43 | import hudson.ExtensionList; 44 | import hudson.Util; 45 | import hudson.model.Failure; 46 | import hudson.util.FormValidation; 47 | 48 | import io.jenkins.plugins.artifact_manager_jclouds.JCloudsVirtualFile; 49 | import io.jenkins.plugins.aws.global_configuration.AbstractAwsGlobalConfiguration; 50 | import io.jenkins.plugins.aws.global_configuration.CredentialsAwsGlobalConfiguration; 51 | 52 | import jenkins.model.Jenkins; 53 | import jenkins.security.FIPS140; 54 | import org.jenkinsci.Symbol; 55 | import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; 56 | import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; 57 | import software.amazon.awssdk.regions.Region; 58 | import software.amazon.awssdk.services.s3.S3Client; 59 | import software.amazon.awssdk.services.s3.S3ClientBuilder; 60 | import software.amazon.awssdk.services.s3.model.Bucket; 61 | import software.amazon.awssdk.services.s3.model.CreateBucketRequest; 62 | import software.amazon.awssdk.services.s3.model.CreateBucketResponse; 63 | import software.amazon.awssdk.services.s3.model.GetBucketLocationRequest; 64 | 65 | /** 66 | * Store the S3BlobStore configuration to save it on a separate file. This make that 67 | * the change of container does not affected to the Artifact functionality, you could change the container 68 | * and it would still work if both container contains the same data. 69 | */ 70 | @Symbol("s3") 71 | @Extension 72 | public final class S3BlobStoreConfig extends AbstractAwsGlobalConfiguration { 73 | 74 | private static final String BUCKET_REGEXP = "^([a-z]|(\\d(?!\\d{0,2}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})))([a-z\\d]|(\\.(?!(\\.|-)))|(-(?!\\.))){1,61}[a-z\\d\\.]$"; 75 | private static final Pattern bucketPattern = Pattern.compile(BUCKET_REGEXP); 76 | 77 | private static final String ENDPOINT_REGEXP = "^[a-z0-9][a-z0-9-.]{0,}(?::[0-9]{1,5})?$"; 78 | private static final Pattern endPointPattern = Pattern.compile(ENDPOINT_REGEXP, Pattern.CASE_INSENSITIVE); 79 | 80 | @SuppressWarnings("FieldMayBeFinal") 81 | private static boolean DELETE_ARTIFACTS = Boolean.getBoolean(S3BlobStoreConfig.class.getName() + ".deleteArtifacts"); 82 | @SuppressWarnings("FieldMayBeFinal") 83 | private static boolean DELETE_STASHES = Boolean.getBoolean(S3BlobStoreConfig.class.getName() + ".deleteStashes"); 84 | 85 | /** 86 | * Name of the S3 Bucket. 87 | */ 88 | private String container; 89 | /** 90 | * Prefix to use for files, use to be a folder. 91 | */ 92 | private String prefix; 93 | @Deprecated private transient String region, credentialsId; 94 | 95 | private boolean usePathStyleUrl; 96 | 97 | private boolean useHttp; 98 | 99 | private boolean useTransferAcceleration; 100 | 101 | private boolean disableSessionToken; 102 | 103 | private String customEndpoint; 104 | 105 | private String customSigningRegion; 106 | 107 | private final boolean deleteArtifacts; 108 | 109 | private final boolean deleteStashes; 110 | 111 | /** 112 | * class to test configuration against Amazon S3 Bucket. 113 | */ 114 | private static class S3BlobStoreTester extends S3BlobStore { 115 | private static final long serialVersionUID = -3645770416235883487L; 116 | 117 | @SuppressFBWarnings(value = "SE_TRANSIENT_FIELD_NOT_RESTORED", justification = "This transient field is only modified from the class constructor.") 118 | private transient S3BlobStoreConfig config; 119 | 120 | S3BlobStoreTester(String container, String prefix, boolean useHttp, 121 | boolean useTransferAcceleration, boolean usePathStyleUrl, 122 | boolean disableSessionToken, String customEndpoint, 123 | String customSigningRegion) { 124 | config = new S3BlobStoreConfig(); 125 | config.setContainer(container); 126 | config.setPrefix(prefix); 127 | config.setCustomEndpoint(customEndpoint); 128 | config.setCustomSigningRegion(customSigningRegion); 129 | config.setUseHttp(useHttp); 130 | config.setUseTransferAcceleration(useTransferAcceleration); 131 | config.setUsePathStyleUrl(usePathStyleUrl); 132 | config.setDisableSessionToken(disableSessionToken); 133 | } 134 | 135 | @Override 136 | public S3BlobStoreConfig getConfiguration() { 137 | return config; 138 | } 139 | } 140 | 141 | public S3BlobStoreConfig() { 142 | load(); 143 | if (Util.fixEmpty(region) != null || Util.fixEmpty(credentialsId) != null) { 144 | CredentialsAwsGlobalConfiguration.get().setRegion(region); 145 | CredentialsAwsGlobalConfiguration.get().setCredentialsId(credentialsId); 146 | region = null; 147 | credentialsId = null; 148 | save(); 149 | } 150 | deleteArtifacts = DELETE_ARTIFACTS; 151 | deleteStashes = DELETE_STASHES; 152 | } 153 | 154 | public String getContainer() { 155 | return container; 156 | } 157 | 158 | @DataBoundSetter 159 | public void setContainer(String container) { 160 | this.container = container; 161 | checkValue(doCheckContainer(container)); 162 | save(); 163 | } 164 | 165 | public String getPrefix() { 166 | return prefix; 167 | } 168 | 169 | @DataBoundSetter 170 | public void setPrefix(String prefix){ 171 | this.prefix = prefix; 172 | checkValue(doCheckPrefix(prefix)); 173 | save(); 174 | } 175 | 176 | private void checkValue(@NonNull FormValidation formValidation) { 177 | if (formValidation.kind == FormValidation.Kind.ERROR) { 178 | throw new Failure(formValidation.getMessage()); 179 | } 180 | } 181 | 182 | public boolean isDeleteArtifacts() { 183 | return deleteArtifacts; 184 | } 185 | 186 | public boolean isDeleteStashes() { 187 | return deleteStashes; 188 | } 189 | 190 | public boolean getUsePathStyleUrl() { 191 | return usePathStyleUrl; 192 | } 193 | 194 | @DataBoundSetter 195 | public void setUsePathStyleUrl(boolean usePathStyleUrl){ 196 | this.usePathStyleUrl = usePathStyleUrl; 197 | save(); 198 | } 199 | 200 | public boolean getUseHttp() { 201 | return useHttp; 202 | } 203 | 204 | @DataBoundSetter 205 | public void setUseHttp(boolean useHttp) { 206 | checkValue(doCheckUseHttp(useHttp)); 207 | this.useHttp = useHttp; 208 | save(); 209 | } 210 | 211 | public boolean getUseTransferAcceleration() { 212 | return useTransferAcceleration; 213 | } 214 | 215 | @DataBoundSetter 216 | public void setUseTransferAcceleration(boolean useTransferAcceleration){ 217 | this.useTransferAcceleration = useTransferAcceleration; 218 | save(); 219 | } 220 | 221 | public boolean getDisableSessionToken() { 222 | return disableSessionToken; 223 | } 224 | 225 | @DataBoundSetter 226 | public void setDisableSessionToken(boolean disableSessionToken){ 227 | this.disableSessionToken = disableSessionToken; 228 | save(); 229 | } 230 | 231 | public String getCustomEndpoint() { 232 | return customEndpoint; 233 | } 234 | 235 | @DataBoundSetter 236 | public void setCustomEndpoint(String customEndpoint){ 237 | checkValue(doCheckCustomEndpoint(customEndpoint)); 238 | this.customEndpoint = customEndpoint; 239 | save(); 240 | } 241 | 242 | public String getResolvedCustomEndpoint() { 243 | if(StringUtils.isNotBlank(customEndpoint)) { 244 | String protocol; 245 | if(getUseHttp()) { 246 | protocol = "http"; 247 | } else { 248 | protocol = "https"; 249 | } 250 | return protocol + "://" + customEndpoint; 251 | } 252 | return null; 253 | } 254 | 255 | public String getCustomSigningRegion() { 256 | return customSigningRegion; 257 | } 258 | 259 | @DataBoundSetter 260 | public void setCustomSigningRegion(String customSigningRegion){ 261 | this.customSigningRegion = customSigningRegion; 262 | checkValue(doCheckCustomSigningRegion(this.customSigningRegion)); 263 | save(); 264 | } 265 | 266 | @NonNull 267 | @Override 268 | public String getDisplayName() { 269 | return "Artifact Manager Amazon S3 Bucket"; 270 | } 271 | 272 | @NonNull 273 | public static S3BlobStoreConfig get() { 274 | return ExtensionList.lookupSingleton(S3BlobStoreConfig.class); 275 | } 276 | 277 | /** 278 | * 279 | * @return an AmazonS3ClientBuilder using the region or not, it depends if a region is configured or not. 280 | */ 281 | S3ClientBuilder getAmazonS3ClientBuilder() throws URISyntaxException { 282 | S3ClientBuilder ret = S3Client.builder(); 283 | 284 | if (StringUtils.isNotBlank(getResolvedCustomEndpoint())) { 285 | String resolvedCustomSigningRegion = customSigningRegion; 286 | if (StringUtils.isBlank(resolvedCustomSigningRegion)) { 287 | resolvedCustomSigningRegion = "us-east-1"; 288 | } 289 | ret = ret.endpointOverride(new URI(getResolvedCustomEndpoint())).region(Region.of(resolvedCustomSigningRegion)); 290 | } else if (StringUtils.isNotBlank(CredentialsAwsGlobalConfiguration.get().getRegion())) { 291 | ret = ret.region(Region.of(CredentialsAwsGlobalConfiguration.get().getRegion())); 292 | } else { 293 | ret = ret.useArnRegion(true); 294 | } 295 | ret = ret.accelerate(useTransferAcceleration); 296 | 297 | // TODO the client would automatically use path-style URLs under certain conditions; is it really necessary to override? 298 | ret = ret.forcePathStyle(getUsePathStyleUrl()); 299 | 300 | return ret; 301 | } 302 | 303 | @VisibleForTesting 304 | public S3ClientBuilder getAmazonS3ClientBuilderWithCredentials() throws IOException { 305 | try { 306 | return getAmazonS3ClientBuilderWithCredentials(getDisableSessionToken()); 307 | } catch (URISyntaxException e) { 308 | throw new RuntimeException(e); 309 | } 310 | } 311 | 312 | private S3ClientBuilder getAmazonS3ClientBuilderWithCredentials(boolean disableSessionToken) throws IOException, URISyntaxException { 313 | S3ClientBuilder builder = getAmazonS3ClientBuilder(); 314 | if (disableSessionToken) { 315 | builder = builder.credentialsProvider(CredentialsAwsGlobalConfiguration.get().getCredentials()); 316 | } else { 317 | AwsSessionCredentials awsSessionCredentials = CredentialsAwsGlobalConfiguration.get() 318 | .sessionCredentials(CredentialsAwsGlobalConfiguration.get().getRegion(), 319 | CredentialsAwsGlobalConfiguration.get().getCredentialsId()); 320 | if(awsSessionCredentials != null ) { 321 | builder.credentialsProvider(StaticCredentialsProvider.create(awsSessionCredentials)); 322 | } else { 323 | throw new IOException("No session AWS credentials found"); 324 | } 325 | } 326 | return builder; 327 | } 328 | 329 | public FormValidation doCheckContainer(@QueryParameter String container){ 330 | FormValidation ret = FormValidation.ok(); 331 | if (StringUtils.isBlank(container)){ 332 | ret = FormValidation.warning("The container name cannot be empty"); 333 | } else if (!bucketPattern.matcher(container).matches()){ 334 | ret = FormValidation.error("The S3 Bucket name does not match S3 bucket rules"); 335 | } 336 | return ret; 337 | } 338 | 339 | public FormValidation doCheckPrefix(@QueryParameter String prefix){ 340 | FormValidation ret; 341 | if (StringUtils.isBlank(prefix)) { 342 | ret = FormValidation.ok("Artifacts will be stored in the root folder of the S3 Bucket."); 343 | } else if (prefix.endsWith("/")) { 344 | ret = FormValidation.ok(); 345 | } else { 346 | ret = FormValidation.error("A prefix must end with a slash."); 347 | } 348 | return ret; 349 | } 350 | 351 | public FormValidation doCheckCustomSigningRegion(@QueryParameter String customSigningRegion) { 352 | FormValidation ret; 353 | if (StringUtils.isBlank(customSigningRegion) && StringUtils.isNotBlank(customEndpoint)) { 354 | ret = FormValidation.ok("'us-east-1' will be used when a custom endpoint is configured and custom signing region is blank."); 355 | } else { 356 | ret = FormValidation.ok(); 357 | } 358 | return ret; 359 | } 360 | 361 | public FormValidation doCheckCustomEndpoint(@QueryParameter String customEndpoint) { 362 | FormValidation ret = FormValidation.ok(); 363 | if (!StringUtils.isBlank(customEndpoint) && !endPointPattern.matcher(customEndpoint).matches()) { 364 | ret = FormValidation.error("Custom Endpoint may not be valid."); 365 | } 366 | return ret; 367 | } 368 | 369 | public FormValidation doCheckUseHttp(@QueryParameter boolean useHttp) { 370 | Jenkins.get().checkPermission(Jenkins.ADMINISTER); 371 | if (FIPS140.useCompliantAlgorithms() && useHttp) { 372 | return FormValidation.error("Cannot use HTTP in FIPS mode."); 373 | } 374 | return FormValidation.ok(); 375 | } 376 | 377 | /** 378 | * create an S3 Bucket. 379 | * @param name name of the S3 Bucket. 380 | * @return return the Bucket created. 381 | * @throws IOException in case of error obtaining the credentials, in other kind of errors it will throw the 382 | * runtime exceptions are thrown by createBucket method. 383 | */ 384 | public Bucket createS3Bucket(String name) throws IOException { 385 | try { 386 | return createS3Bucket(name, getDisableSessionToken()); 387 | } catch (URISyntaxException e) { 388 | throw new IOException(e); 389 | } 390 | } 391 | 392 | private Bucket createS3Bucket(String name, boolean disableSessionToken) throws IOException, URISyntaxException { 393 | S3ClientBuilder builder = getAmazonS3ClientBuilderWithCredentials(disableSessionToken); 394 | //Accelerated mode must be off in order to apply it to a bucket 395 | try (S3Client client = builder.accelerate(false).build()) { 396 | CreateBucketResponse response = client.createBucket(CreateBucketRequest.builder().bucket(name).build()); 397 | if(response.sdkHttpResponse().isSuccessful()) { 398 | return Bucket.builder().name(name).build(); 399 | } else { 400 | throw new IOException("Cannot create bucket with name:" + name 401 | + " response status : " + response.sdkHttpResponse().statusCode()); 402 | } 403 | } 404 | } 405 | 406 | @RequirePOST 407 | public FormValidation doCreateS3Bucket(@QueryParameter String container, @QueryParameter boolean disableSessionToken) { 408 | Jenkins.get().checkPermission(Jenkins.ADMINISTER); 409 | FormValidation ret = FormValidation.ok("success"); 410 | try { 411 | createS3Bucket(container, disableSessionToken); 412 | } catch (Throwable t){ 413 | String msg = processExceptionMessage(t); 414 | ret = FormValidation.error(StringUtils.abbreviate(msg, 200)); 415 | } 416 | return ret; 417 | } 418 | 419 | void checkGetBucketLocation(String container, boolean disableSessionToken) throws IOException, URISyntaxException { 420 | S3ClientBuilder builder = getAmazonS3ClientBuilderWithCredentials(disableSessionToken); 421 | try (S3Client client = builder.build()) { 422 | client.getBucketLocation(GetBucketLocationRequest.builder().bucket(container).build()); 423 | } 424 | } 425 | 426 | @RequirePOST 427 | public FormValidation doValidateS3BucketConfig( 428 | @QueryParameter String container, 429 | @QueryParameter String prefix, 430 | @QueryParameter boolean useHttp, 431 | @QueryParameter boolean useTransferAcceleration, 432 | @QueryParameter boolean usePathStyleUrl, 433 | @QueryParameter boolean disableSessionToken, 434 | @QueryParameter String customEndpoint, 435 | @QueryParameter String customSigningRegion) { 436 | Jenkins.get().checkPermission(Jenkins.ADMINISTER); 437 | 438 | if (FIPS140.useCompliantAlgorithms() && useHttp) { 439 | return FormValidation.warning("Validation failed as \"use Insecure Http\" flag is enabled while in FIPS mode"); 440 | } 441 | FormValidation ret = FormValidation.ok("success"); 442 | S3BlobStore provider = new S3BlobStoreTester(container, prefix, 443 | useHttp, useTransferAcceleration,usePathStyleUrl, 444 | disableSessionToken, customEndpoint, customSigningRegion); 445 | 446 | try { 447 | JCloudsVirtualFile jc = new JCloudsVirtualFile(provider, container, prefix.replaceFirst("/$", "")); 448 | jc.list(); 449 | } catch (Throwable t){ 450 | String msg = processExceptionMessage(t); 451 | ret = FormValidation.error(t, StringUtils.abbreviate(msg, 200)); 452 | } 453 | try { 454 | provider.getConfiguration().checkGetBucketLocation(container, disableSessionToken); 455 | } catch (Throwable t){ 456 | ret = FormValidation.warning(t, "GetBucketLocation failed"); 457 | } 458 | return ret; 459 | } 460 | 461 | } 462 | -------------------------------------------------------------------------------- /src/main/resources/index.jelly: -------------------------------------------------------------------------------- 1 | 2 | 3 |
4 | A Jenkins plugin to keep artifacts and Pipeline stashes in Amazon S3. 5 |
6 | -------------------------------------------------------------------------------- /src/main/resources/io/jenkins/plugins/artifact_manager_jclouds/JCloudsArtifactManagerFactory/config.jelly: -------------------------------------------------------------------------------- 1 | 2 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /src/main/resources/io/jenkins/plugins/artifact_manager_jclouds/s3/S3BlobStore/config.jelly: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 |
7 | ${%configure_first} 8 |
9 |
10 |
11 |
12 | -------------------------------------------------------------------------------- /src/main/resources/io/jenkins/plugins/artifact_manager_jclouds/s3/S3BlobStore/config.properties: -------------------------------------------------------------------------------- 1 | configure_first=First configure Amazon settings. 2 | -------------------------------------------------------------------------------- /src/main/resources/io/jenkins/plugins/artifact_manager_jclouds/s3/S3BlobStoreConfig/config.jelly: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 37 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /src/main/resources/io/jenkins/plugins/artifact_manager_jclouds/s3/S3BlobStoreConfig/help-container.html: -------------------------------------------------------------------------------- 1 |
Name of the S3 Bucket, the S3 Bucket should exists and the AWS account/profile/role used to access should have access to it
-------------------------------------------------------------------------------- /src/main/resources/io/jenkins/plugins/artifact_manager_jclouds/s3/S3BlobStoreConfig/help-customEndpoint.html: -------------------------------------------------------------------------------- 1 |
Use this to set a custom S3 service endpoint. This option is used if you're using an S3 compatible service, leave it blank if you're using AWS's S3 (s3.amazonaws.com)
2 | -------------------------------------------------------------------------------- /src/main/resources/io/jenkins/plugins/artifact_manager_jclouds/s3/S3BlobStoreConfig/help-customSigningRegion.html: -------------------------------------------------------------------------------- 1 |
Custom signing region to be used with the custom endpoint if set
2 | -------------------------------------------------------------------------------- /src/main/resources/io/jenkins/plugins/artifact_manager_jclouds/s3/S3BlobStoreConfig/help-deleteArtifacts.html: -------------------------------------------------------------------------------- 1 |
This parameter is set by the property -Dio.jenkins.plugins.artifact_manager_jclouds.s3.S3BlobStoreConfig.deleteArtifacts=true, 2 | if not checked the artifacts will not be deleted from S3 when the corresponding build is deleted. 3 |
4 | -------------------------------------------------------------------------------- /src/main/resources/io/jenkins/plugins/artifact_manager_jclouds/s3/S3BlobStoreConfig/help-deleteStashes.html: -------------------------------------------------------------------------------- 1 |
This parameter is set by the property -Dio.jenkins.plugins.artifact_manager_jclouds.s3.S3BlobStoreConfig.deleteStashes=true, 2 | if not checked the stash will not be deleted from S3 when the corresponding build is deleted. 3 |
4 | -------------------------------------------------------------------------------- /src/main/resources/io/jenkins/plugins/artifact_manager_jclouds/s3/S3BlobStoreConfig/help-disableSessionToken.html: -------------------------------------------------------------------------------- 1 |
Disable session tokens for S3 requests. Session tokens will query AWS and so will not work if you're using an S3 compatible service.
2 | -------------------------------------------------------------------------------- /src/main/resources/io/jenkins/plugins/artifact_manager_jclouds/s3/S3BlobStoreConfig/help-prefix.html: -------------------------------------------------------------------------------- 1 |
This is the string that will be used as prefix for every file path, it must end with an "/" if it is a folder.
-------------------------------------------------------------------------------- /src/main/resources/io/jenkins/plugins/artifact_manager_jclouds/s3/S3BlobStoreConfig/help-useHttp.html: -------------------------------------------------------------------------------- 1 |
Use http for S3 requests. This should only ever be used with an internal S3 compatible service.
2 | -------------------------------------------------------------------------------- /src/main/resources/io/jenkins/plugins/artifact_manager_jclouds/s3/S3BlobStoreConfig/help-usePathStyleUrl.html: -------------------------------------------------------------------------------- 1 |
This option modifies the format of the URL used to communicate with S3.
2 |
3 | When this option is disabled URLs will be formatted https://bucketname.s3.amazonaws.com/key/path
4 | When this option is enabled URLs will be formatted https://s3.amazonaws.com/bucketname/key/path
5 | Typically, you would only ever enable this option when using an S3 compatible service (not AWS S3) that only supported path style URLs. 6 | For Amazon S3 itself, path-style URLs are usually deprecated and may not work at all. 7 |
8 | -------------------------------------------------------------------------------- /src/main/resources/io/jenkins/plugins/artifact_manager_jclouds/s3/S3BlobStoreConfig/help-useTransferAcceleration.html: -------------------------------------------------------------------------------- 1 |
This parameter enables usage of Transfer Acceleration
2 | -------------------------------------------------------------------------------- /src/test/java/io/jenkins/plugins/artifact_manager_jclouds/MockApiMetadata.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2018 CloudBees, Inc. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package io.jenkins.plugins.artifact_manager_jclouds; 26 | 27 | import com.google.inject.AbstractModule; 28 | import java.io.IOException; 29 | import java.net.URI; 30 | import java.util.Collection; 31 | import java.util.HashMap; 32 | import java.util.Map; 33 | import java.util.Set; 34 | import java.util.concurrent.ConcurrentHashMap; 35 | import org.apache.commons.io.IOUtils; 36 | import org.jclouds.apis.ApiMetadata; 37 | import org.jclouds.apis.internal.BaseApiMetadata; 38 | import org.jclouds.blobstore.BlobRequestSigner; 39 | import org.jclouds.blobstore.BlobStore; 40 | import org.jclouds.blobstore.BlobStoreContext; 41 | import org.jclouds.blobstore.LocalBlobRequestSigner; 42 | import org.jclouds.blobstore.LocalStorageStrategy; 43 | import org.jclouds.blobstore.TransientApiMetadata; 44 | import org.jclouds.blobstore.TransientStorageStrategy; 45 | import org.jclouds.blobstore.attr.ConsistencyModel; 46 | import org.jclouds.blobstore.config.BlobStoreObjectModule; 47 | import org.jclouds.blobstore.config.LocalBlobStore; 48 | import org.jclouds.blobstore.config.TransientBlobStoreContextModule; 49 | import org.jclouds.blobstore.domain.Blob; 50 | import org.jclouds.blobstore.domain.BlobAccess; 51 | import org.jclouds.blobstore.domain.ContainerAccess; 52 | import org.jclouds.blobstore.domain.StorageMetadata; 53 | import org.jclouds.blobstore.options.CreateContainerOptions; 54 | import org.jclouds.blobstore.options.ListContainerOptions; 55 | import org.jclouds.domain.Location; 56 | import org.jclouds.io.Payloads; 57 | import org.kohsuke.MetaInfServices; 58 | 59 | /** 60 | * A mock provider allowing control of all operations. 61 | * Otherwise akin to a simplified version of {@link TransientApiMetadata}. 62 | * Whereas the stock {@code transient} provider would merely implement the full SPI, 63 | * we also want to allow particular metadata operations to fail or block at test-specified times. 64 | */ 65 | @MetaInfServices(ApiMetadata.class) 66 | public final class MockApiMetadata extends BaseApiMetadata { 67 | 68 | public MockApiMetadata() { 69 | this(new Builder()); 70 | } 71 | 72 | private MockApiMetadata(Builder builder) { 73 | super(builder); 74 | } 75 | 76 | @Override 77 | public Builder toBuilder() { 78 | return new Builder().fromApiMetadata(this); 79 | } 80 | 81 | private static final class Builder extends BaseApiMetadata.Builder { 82 | 83 | Builder() { 84 | id("mock"); 85 | name("mock"); 86 | identityName("mock"); 87 | documentation(URI.create("about:nothing")); 88 | defaultIdentity("nobody"); 89 | defaultCredential("anon"); 90 | defaultEndpoint("http://nowhere.net/"); 91 | view(BlobStoreContext.class); 92 | defaultModule(MockModule.class); 93 | } 94 | 95 | @Override 96 | protected Builder self() { 97 | return this; 98 | } 99 | 100 | @Override 101 | public ApiMetadata build() { 102 | return new MockApiMetadata(this); 103 | } 104 | 105 | } 106 | 107 | /** Like {@link TransientBlobStoreContextModule}. */ 108 | public static final class MockModule extends AbstractModule { 109 | 110 | @Override 111 | protected void configure() { 112 | install(new BlobStoreObjectModule()); 113 | bind(BlobStore.class).to(LocalBlobStore.class); 114 | bind(ConsistencyModel.class).toInstance(ConsistencyModel.STRICT); 115 | bind(LocalStorageStrategy.class).to(MockStrategy.class); 116 | bind(BlobRequestSigner.class).to(LocalBlobRequestSigner.class); 117 | } 118 | 119 | } 120 | 121 | @FunctionalInterface 122 | interface GetBlobKeysInsideContainerHandler { 123 | void run() throws IOException; 124 | } 125 | 126 | private static final Map getBlobKeysInsideContainerHandlers = new ConcurrentHashMap<>(); 127 | 128 | static void handleGetBlobKeysInsideContainer(String container, GetBlobKeysInsideContainerHandler handler) { 129 | getBlobKeysInsideContainerHandlers.put(container, handler); 130 | } 131 | 132 | private static final Map removeBlobHandlers = new ConcurrentHashMap<>(); 133 | 134 | static void handleRemoveBlob(String container, String key, Runnable handler) { 135 | removeBlobHandlers.put(container + '/' + key, handler); 136 | } 137 | 138 | /** Like {@link TransientStorageStrategy}. */ 139 | public static final class MockStrategy implements LocalStorageStrategy { 140 | 141 | private final Map> blobsByContainer = new HashMap<>(); 142 | 143 | @Override 144 | public boolean containerExists(String container) { 145 | return blobsByContainer.containsKey(container); 146 | } 147 | 148 | @Override 149 | public Collection getAllContainerNames() { 150 | return blobsByContainer.keySet(); 151 | } 152 | 153 | @Override 154 | public boolean createContainerInLocation(String container, Location location, CreateContainerOptions options) { 155 | return blobsByContainer.putIfAbsent(container, new HashMap<>()) == null; 156 | } 157 | 158 | @Override 159 | public ContainerAccess getContainerAccess(String container) { 160 | throw new UnsupportedOperationException(); // TODO 161 | } 162 | 163 | @Override 164 | public void setContainerAccess(String container, ContainerAccess access) { 165 | throw new UnsupportedOperationException(); // TODO 166 | } 167 | 168 | @Override 169 | public void deleteContainer(String container) { 170 | blobsByContainer.remove(container); 171 | } 172 | 173 | @Override 174 | public void clearContainer(String container) { 175 | blobsByContainer.get(container).clear(); 176 | } 177 | 178 | @Override 179 | public void clearContainer(String container, ListContainerOptions options) { 180 | throw new UnsupportedOperationException(); // TODO 181 | } 182 | 183 | @Override 184 | public StorageMetadata getContainerMetadata(String container) { 185 | throw new UnsupportedOperationException(); // TODO 186 | } 187 | 188 | @Override 189 | public boolean blobExists(String container, String key) { 190 | return blobsByContainer.get(container).containsKey(key); 191 | } 192 | 193 | @Override 194 | public Iterable getBlobKeysInsideContainer(String container, String prefix, String delimiter) throws IOException { 195 | GetBlobKeysInsideContainerHandler handler = getBlobKeysInsideContainerHandlers.remove(container); 196 | if (handler != null) { 197 | handler.run(); 198 | throw new AssertionError("not supposed to get here"); 199 | } 200 | Set keys = blobsByContainer.get(container).keySet(); 201 | return prefix == null ? keys : () -> keys.stream().filter(path -> path.startsWith(prefix)).iterator(); 202 | } 203 | 204 | @Override 205 | public Blob getBlob(String containerName, String blobName) { 206 | Blob blob = blobsByContainer.get(containerName).get(blobName); 207 | assert blob == null || containerName.equals(blob.getMetadata().getContainer()) : blob; 208 | return blob; 209 | } 210 | 211 | @SuppressWarnings("deprecation") 212 | @Override 213 | public String putBlob(String containerName, Blob blob) throws IOException { 214 | { 215 | // When called from LocalBlobStore.copyBlob, there is no container, and it uses an InputStreamPayload which cannot be reused. 216 | // TransientStorageStrategy has an elaborate createUpdatedCopyOfBlobInContainer here, but these two fixups seem to suffice. 217 | blob.getMetadata().setContainer(containerName); 218 | byte[] data = IOUtils.toByteArray(blob.getPayload().openStream()); 219 | blob.getMetadata().setSize((long) data.length); 220 | blob.setPayload(Payloads.newByteArrayPayload(data)); 221 | } 222 | blobsByContainer.get(containerName).put(blob.getMetadata().getName(), blob); 223 | return null; 224 | } 225 | 226 | @Override 227 | public String putBlob(String containerName, Blob blob, BlobAccess ba) throws IOException { 228 | return putBlob(containerName, blob); 229 | } 230 | 231 | @Override 232 | public void removeBlob(String container, String key) { 233 | Runnable handler = removeBlobHandlers.remove(container + '/' + key); 234 | if (handler != null) { 235 | handler.run(); 236 | return; 237 | } 238 | blobsByContainer.get(container).remove(key); 239 | } 240 | 241 | @Override 242 | public BlobAccess getBlobAccess(String container, String key) { 243 | return BlobAccess.PRIVATE; 244 | } 245 | 246 | @Override 247 | public void setBlobAccess(String container, String key, BlobAccess access) { 248 | // ignore 249 | } 250 | 251 | @Override 252 | public Location getLocation(String containerName) { 253 | return null; 254 | } 255 | 256 | @Override 257 | public String getSeparator() { 258 | return "/"; 259 | } 260 | 261 | } 262 | 263 | } 264 | -------------------------------------------------------------------------------- /src/test/java/io/jenkins/plugins/artifact_manager_jclouds/MockApiMetadataTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2018 CloudBees, Inc. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package io.jenkins.plugins.artifact_manager_jclouds; 25 | 26 | import java.util.stream.Collectors; 27 | import org.jclouds.ContextBuilder; 28 | import org.jclouds.blobstore.BlobStore; 29 | import org.jclouds.blobstore.BlobStoreContext; 30 | import org.jclouds.blobstore.domain.Blob; 31 | import org.jclouds.blobstore.domain.StorageMetadata; 32 | import static org.junit.Assert.*; 33 | import org.junit.Test; 34 | 35 | public class MockApiMetadataTest { 36 | 37 | @Test 38 | public void smokes() throws Exception { 39 | BlobStoreContext bsc = ContextBuilder.newBuilder("mock").buildView(BlobStoreContext.class); 40 | BlobStore bs = bsc.getBlobStore(); 41 | bs.createContainerInLocation(null, "container"); 42 | Blob blob = bs.blobBuilder("file.txt").payload("content").build(); 43 | bs.putBlob("container", blob); 44 | assertEquals("file.txt", bs.list("container").stream().map(StorageMetadata::getName).collect(Collectors.joining(":"))); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/test/java/io/jenkins/plugins/artifact_manager_jclouds/MockBlobStore.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2018 CloudBees, Inc. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package io.jenkins.plugins.artifact_manager_jclouds; 26 | 27 | import java.io.IOException; 28 | import java.net.URI; 29 | import java.net.URL; 30 | import java.util.Map; 31 | import java.util.concurrent.ConcurrentHashMap; 32 | import java.util.logging.Level; 33 | import java.util.logging.Logger; 34 | import java.util.regex.Matcher; 35 | import java.util.regex.Pattern; 36 | import org.apache.commons.io.IOUtils; 37 | import org.apache.http.ConnectionClosedException; 38 | import org.apache.http.HttpEntity; 39 | import org.apache.http.HttpEntityEnclosingRequest; 40 | import org.apache.http.HttpRequest; 41 | import org.apache.http.HttpResponse; 42 | import org.apache.http.entity.ByteArrayEntity; 43 | import org.apache.http.impl.bootstrap.HttpServer; 44 | import org.apache.http.impl.bootstrap.ServerBootstrap; 45 | import org.apache.http.protocol.HttpContext; 46 | import org.apache.http.protocol.HttpRequestHandler; 47 | import org.jclouds.ContextBuilder; 48 | import org.jclouds.blobstore.BlobStore; 49 | import org.jclouds.blobstore.BlobStoreContext; 50 | import org.jclouds.blobstore.domain.Blob; 51 | import org.jclouds.blobstore.domain.StorageMetadata; 52 | 53 | /** 54 | * A mock storage provider which keeps all blobs in memory. 55 | * Presigned “external” URLs are supported. 56 | * Allows tests to inject failures such as HTTP errors or hangs. 57 | */ 58 | public final class MockBlobStore extends BlobStoreProvider { 59 | 60 | private static final long serialVersionUID = 42L; 61 | private static final Logger LOGGER = Logger.getLogger(MockBlobStore.class.getName()); 62 | 63 | private transient BlobStoreContext context; 64 | private transient URL baseURL; 65 | 66 | @Override 67 | public String getPrefix() { 68 | return ""; 69 | } 70 | 71 | @Override 72 | public String getContainer() { 73 | return "container"; 74 | } 75 | 76 | private static final Map specialHandlers = new ConcurrentHashMap<>(); 77 | 78 | /** 79 | * Requests that the next HTTP access to a particular presigned URL should behave specially. 80 | * @param method upload or download 81 | * @param key the blob’s {@link StorageMetadata#getName} 82 | * @param handler what to do instead 83 | */ 84 | static void speciallyHandle(HttpMethod method, String key, HttpRequestHandler handler) { 85 | specialHandlers.put(method + ":" + key, handler); 86 | } 87 | 88 | @Override 89 | public synchronized BlobStoreContext getContext() throws IOException { 90 | if (context == null) { 91 | context = ContextBuilder.newBuilder("mock").buildView(BlobStoreContext.class); 92 | HttpServer server = ServerBootstrap.bootstrap(). 93 | registerHandler("*", (HttpRequest request, HttpResponse response, HttpContext _context) -> { 94 | String method = request.getRequestLine().getMethod(); 95 | Matcher m = Pattern.compile("/([^/]+)/(.+)[?]method=" + method).matcher(request.getRequestLine().getUri()); 96 | if (!m.matches()) { 97 | throw new IllegalStateException(); 98 | } 99 | String container = m.group(1); 100 | String key = m.group(2); 101 | HttpRequestHandler specialHandler = specialHandlers.remove(method + ":" + key); 102 | if (specialHandler != null) { 103 | specialHandler.handle(request, response, _context); 104 | return; 105 | } 106 | BlobStore blobStore = context.getBlobStore(); 107 | switch (method) { 108 | case "GET": { 109 | Blob blob = blobStore.getBlob(container, key); 110 | if (blob == null) { 111 | response.setStatusCode(404); 112 | return; 113 | } 114 | byte[] data = IOUtils.toByteArray(blob.getPayload().openStream()); 115 | response.setStatusCode(200); 116 | response.setEntity(new ByteArrayEntity(data)); 117 | LOGGER.log(Level.INFO, "Serving {0} bytes from {1}:{2}", new Object[] {data.length, container, key}); 118 | return; 119 | } case "PUT": { 120 | HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity(); 121 | byte[] data = IOUtils.toByteArray(entity.getContent()); 122 | Blob blob = blobStore.blobBuilder(key).payload(data).build(); 123 | if (!blobStore.containerExists(container)) { 124 | blobStore.createContainerInLocation(null, container); 125 | } 126 | blobStore.putBlob(container, blob); 127 | response.setStatusCode(204); 128 | LOGGER.log(Level.INFO, "Uploaded {0} bytes to {1}:{2}", new Object[] {data.length, container, key}); 129 | return; 130 | } default: { 131 | throw new IllegalStateException(); 132 | } 133 | } 134 | }). 135 | setExceptionLogger(x -> { 136 | if (x instanceof ConnectionClosedException) { 137 | LOGGER.info(x.toString()); 138 | } else { 139 | LOGGER.log(Level.INFO, "error thrown in HTTP service", x); 140 | } 141 | }). 142 | create(); 143 | server.start(); 144 | baseURL = new URL("http", server.getInetAddress().getHostName(), server.getLocalPort(), "/"); 145 | LOGGER.log(Level.INFO, "Mock server running at {0}", baseURL); 146 | } 147 | return context; 148 | } 149 | 150 | @Override 151 | public URI toURI(String container, String key) { 152 | return URI.create("mock://" + container + "/" + key); 153 | } 154 | 155 | @Override 156 | public URL toExternalURL(Blob blob, HttpMethod httpMethod) throws IOException { 157 | return new URL(baseURL, blob.getMetadata().getContainer() + "/" + blob.getMetadata().getName() + "?method=" + httpMethod); 158 | } 159 | 160 | @Override 161 | public boolean isDeleteArtifacts() { 162 | return true; 163 | } 164 | 165 | @Override 166 | public boolean isDeleteStashes() { 167 | return true; 168 | } 169 | 170 | } 171 | -------------------------------------------------------------------------------- /src/test/java/io/jenkins/plugins/artifact_manager_jclouds/MockBlobStoreTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2018 CloudBees, Inc. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package io.jenkins.plugins.artifact_manager_jclouds; 26 | 27 | import org.jenkinsci.plugins.workflow.ArtifactManagerTest; 28 | import org.junit.ClassRule; 29 | import org.junit.Rule; 30 | import org.junit.Test; 31 | import org.jvnet.hudson.test.BuildWatcher; 32 | import org.jvnet.hudson.test.JenkinsRule; 33 | 34 | public class MockBlobStoreTest { 35 | 36 | @ClassRule 37 | public static BuildWatcher buildWatcher = new BuildWatcher(); 38 | 39 | @Rule 40 | public JenkinsRule j = new JenkinsRule(); 41 | 42 | @Test 43 | public void smokes() throws Exception { 44 | ArtifactManagerTest.artifactArchiveAndDelete(j, new JCloudsArtifactManagerFactory(new MockBlobStore()), false, null); 45 | ArtifactManagerTest.artifactStashAndDelete(j, new JCloudsArtifactManagerFactory(new MockBlobStore()), false, null); 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/test/java/io/jenkins/plugins/artifact_manager_jclouds/s3/AbstractIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package io.jenkins.plugins.artifact_manager_jclouds.s3; 2 | 3 | import io.jenkins.plugins.artifact_manager_jclouds.JCloudsArtifactManagerFactory; 4 | import java.io.IOException; 5 | import jenkins.model.ArtifactManagerFactory; 6 | import org.jenkinsci.plugins.workflow.ArtifactManagerTest; 7 | import org.junit.Rule; 8 | import org.junit.Test; 9 | import org.jvnet.hudson.test.JenkinsRule; 10 | import org.jvnet.hudson.test.LoggerRule; 11 | import org.jvnet.hudson.test.RealJenkinsRule; 12 | import software.amazon.awssdk.services.s3.S3Client; 13 | import software.amazon.awssdk.services.s3.model.Bucket; 14 | import software.amazon.awssdk.services.s3.model.CreateBucketRequest; 15 | import software.amazon.awssdk.services.s3.model.HeadBucketRequest; 16 | 17 | import java.util.logging.Level; 18 | 19 | import static org.hamcrest.MatcherAssert.assertThat; 20 | import static org.hamcrest.Matchers.is; 21 | import static org.junit.Assert.assertEquals; 22 | import org.junit.Assume; 23 | import org.junit.BeforeClass; 24 | import org.testcontainers.DockerClientFactory; 25 | 26 | public abstract class AbstractIntegrationTest { 27 | 28 | protected static final String CONTAINER_NAME = "jenkins"; 29 | protected static final String CONTAINER_PREFIX = "ci/"; 30 | 31 | @BeforeClass 32 | public static void assumeDocker() throws Exception { 33 | // Beyond just isDockerAvailable, verify the OS: 34 | try { 35 | Assume.assumeThat("expect to run Docker on Linux containers", DockerClientFactory.instance().client().infoCmd().exec().getOsType(), is("linux")); 36 | } catch (Exception x) { 37 | Assume.assumeNoException("does not look like Docker is available", x); 38 | } 39 | } 40 | 41 | private static S3Client client() throws IOException { 42 | return S3BlobStoreConfig.get().getAmazonS3ClientBuilderWithCredentials().build(); 43 | } 44 | 45 | @Rule 46 | public RealJenkinsRule rr = new RealJenkinsRule().javaOptions("-Xmx150m").withDebugPort(8000).withDebugSuspend(true); 47 | 48 | @Rule 49 | public LoggerRule loggerRule = new LoggerRule().recordPackage(JCloudsArtifactManagerFactory.class, Level.FINE); 50 | 51 | protected static ArtifactManagerFactory getArtifactManagerFactory(Boolean deleteArtifacts, Boolean deleteStashes) { 52 | return new JCloudsArtifactManagerFactory(new CustomBehaviorBlobStoreProvider(new S3BlobStore(), deleteArtifacts, deleteStashes)); 53 | } 54 | 55 | protected static void _artifactArchiveAndDelete(JenkinsRule jenkinsRule) throws Throwable { 56 | createBucketWithAwsClient("artifact-archive-and-delete"); 57 | ArtifactManagerTest.artifactArchiveAndDelete(jenkinsRule, getArtifactManagerFactory(true, null), true, null); 58 | } 59 | 60 | protected static void createBucketWithAwsClient(String bucketName) throws IOException { 61 | var config = S3BlobStoreConfig.get(); 62 | config.setContainer(bucketName); 63 | client().createBucket(CreateBucketRequest.builder().bucket(bucketName).build()); 64 | } 65 | 66 | protected static void _artifactArchive(JenkinsRule jenkinsRule) throws Throwable { 67 | createBucketWithAwsClient("artifact-archive"); 68 | assertThat(client().headBucket(HeadBucketRequest.builder().bucket("artifact-archive").build()).sdkHttpResponse().isSuccessful(), is(true)); 69 | ArtifactManagerTest.artifactArchive(jenkinsRule, getArtifactManagerFactory(null, null), true, null); 70 | } 71 | 72 | protected static void _artifactStashAndDelete(JenkinsRule jenkinsRule) throws Throwable { 73 | createBucketWithAwsClient("artifact-stash-and-delete"); 74 | ArtifactManagerTest.artifactStashAndDelete(jenkinsRule, getArtifactManagerFactory(null, true), true, null); 75 | } 76 | 77 | protected static void _canCreateBucket(JenkinsRule r) throws Throwable { 78 | String testBucketName = "jenkins-ci-data"; 79 | var config = S3BlobStoreConfig.get(); 80 | Bucket createdBucket = config.createS3Bucket(testBucketName); 81 | assertEquals(testBucketName, createdBucket.name()); 82 | assertThat(client().headBucket(HeadBucketRequest.builder().bucket(testBucketName).build()).sdkHttpResponse().isSuccessful(), is(true)); 83 | } 84 | 85 | protected static void _artifactStash(JenkinsRule jenkinsRule) throws Throwable { 86 | createBucketWithAwsClient("artifact-stash"); 87 | ArtifactManagerTest.artifactStash(jenkinsRule, getArtifactManagerFactory(null, null), true, null); 88 | } 89 | 90 | @Test 91 | public void canCreateBucket() throws Throwable { 92 | rr.runRemotely(AbstractIntegrationTest::_canCreateBucket); 93 | } 94 | 95 | @Test 96 | public void artifactArchive() throws Throwable { 97 | rr.runRemotely(AbstractIntegrationTest::_artifactArchive); 98 | } 99 | 100 | @Test 101 | public void artifactArchiveAndDelete() throws Throwable { 102 | rr.runRemotely(AbstractIntegrationTest::_artifactArchiveAndDelete); 103 | } 104 | 105 | @Test 106 | public void artifactStash() throws Throwable { 107 | rr.runRemotely(AbstractIntegrationTest::_artifactStash); 108 | } 109 | 110 | @Test 111 | public void artifactStashAndDelete() throws Throwable { 112 | rr.runRemotely(AbstractIntegrationTest::_artifactStashAndDelete); 113 | } 114 | 115 | } 116 | -------------------------------------------------------------------------------- /src/test/java/io/jenkins/plugins/artifact_manager_jclouds/s3/ConfigAsCodeTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2018 CloudBees, Inc. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package io.jenkins.plugins.artifact_manager_jclouds.s3; 26 | 27 | import io.jenkins.plugins.artifact_manager_jclouds.JCloudsArtifactManagerFactory; 28 | import io.jenkins.plugins.aws.global_configuration.CredentialsAwsGlobalConfiguration; 29 | import io.jenkins.plugins.casc.ConfigurationAsCode; 30 | import java.util.List; 31 | import jenkins.model.ArtifactManagerConfiguration; 32 | import jenkins.model.ArtifactManagerFactory; 33 | import static org.junit.Assert.*; 34 | import org.junit.Rule; 35 | import org.junit.Test; 36 | import org.jvnet.hudson.test.Issue; 37 | import org.jvnet.hudson.test.JenkinsRule; 38 | 39 | public class ConfigAsCodeTest { 40 | 41 | @Rule 42 | public JenkinsRule r = new JenkinsRule(); 43 | 44 | @Issue("JENKINS-52304") 45 | @Test 46 | public void smokes() throws Exception { 47 | ConfigurationAsCode.get().configure(ConfigAsCodeTest.class.getResource("configuration-as-code.yml").toString()); 48 | List artifactManagerFactories = ArtifactManagerConfiguration.get().getArtifactManagerFactories(); 49 | assertEquals(1, artifactManagerFactories.size()); 50 | JCloudsArtifactManagerFactory mgr = (JCloudsArtifactManagerFactory) artifactManagerFactories.get(0); 51 | assertEquals(S3BlobStore.class, mgr.getProvider().getClass()); 52 | assertEquals("us-east-1", CredentialsAwsGlobalConfiguration.get().getRegion()); 53 | assertEquals("jenkins_data/", S3BlobStoreConfig.get().getPrefix()); 54 | assertEquals("internal-s3.company.org", S3BlobStoreConfig.get().getCustomEndpoint()); 55 | assertEquals("us-west-2", S3BlobStoreConfig.get().getCustomSigningRegion()); 56 | assertEquals(true, S3BlobStoreConfig.get().getUsePathStyleUrl()); 57 | assertEquals(true, S3BlobStoreConfig.get().getUseHttp()); 58 | assertEquals(true, S3BlobStoreConfig.get().getDisableSessionToken()); 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /src/test/java/io/jenkins/plugins/artifact_manager_jclouds/s3/CustomBehaviorBlobStoreProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2019 CloudBees, Inc. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package io.jenkins.plugins.artifact_manager_jclouds.s3; 26 | 27 | import io.jenkins.plugins.artifact_manager_jclouds.BlobStoreProvider; 28 | import io.jenkins.plugins.artifact_manager_jclouds.BlobStoreProviderDescriptor; 29 | import java.io.IOException; 30 | import java.net.URI; 31 | import java.net.URL; 32 | import org.jclouds.blobstore.BlobStoreContext; 33 | import org.jclouds.blobstore.domain.Blob; 34 | 35 | public class CustomBehaviorBlobStoreProvider extends BlobStoreProvider { 36 | 37 | private final BlobStoreProvider delegate; 38 | private final Boolean deleteArtifacts, deleteStashes; 39 | 40 | CustomBehaviorBlobStoreProvider(BlobStoreProvider delegate, Boolean deleteArtifacts, Boolean deleteStashes) { 41 | this.delegate = delegate; 42 | this.deleteArtifacts = deleteArtifacts; 43 | this.deleteStashes = deleteStashes; 44 | } 45 | 46 | @Override 47 | public String getPrefix() { 48 | return delegate.getPrefix(); 49 | } 50 | 51 | @Override 52 | public String getContainer() { 53 | return delegate.getContainer(); 54 | } 55 | 56 | @Override 57 | public boolean isDeleteArtifacts() { 58 | return deleteArtifacts != null ? deleteArtifacts : delegate.isDeleteArtifacts(); 59 | } 60 | 61 | @Override 62 | public boolean isDeleteStashes() { 63 | return deleteStashes != null ? deleteStashes : delegate.isDeleteStashes(); 64 | } 65 | 66 | @Override 67 | public BlobStoreContext getContext() throws IOException { 68 | return delegate.getContext(); 69 | } 70 | 71 | @Override 72 | public URI toURI(String container, String key) { 73 | return delegate.toURI(container, key); 74 | } 75 | 76 | @Override 77 | public URL toExternalURL(Blob blob, BlobStoreProvider.HttpMethod httpMethod) throws IOException { 78 | return delegate.toExternalURL(blob, httpMethod); 79 | } 80 | 81 | @Override 82 | public BlobStoreProviderDescriptor getDescriptor() { 83 | return delegate.getDescriptor(); 84 | } 85 | 86 | } 87 | -------------------------------------------------------------------------------- /src/test/java/io/jenkins/plugins/artifact_manager_jclouds/s3/JCloudsArtifactManagerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2018 CloudBees, Inc. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package io.jenkins.plugins.artifact_manager_jclouds.s3; 26 | 27 | import io.jenkins.plugins.artifact_manager_jclouds.JCloudsArtifactManagerFactory; 28 | import static org.hamcrest.MatcherAssert.assertThat; 29 | import static org.hamcrest.Matchers.*; 30 | import static org.junit.Assert.assertEquals; 31 | import static org.junit.Assert.assertTrue; 32 | import static org.junit.Assert.fail; 33 | import static org.junit.Assume.*; 34 | 35 | import java.io.IOException; 36 | import java.io.InputStream; 37 | import java.io.OutputStream; 38 | import java.util.logging.Level; 39 | 40 | import org.apache.commons.io.IOUtils; 41 | import org.apache.commons.io.output.NullOutputStream; 42 | import org.jclouds.rest.internal.InvokeHttpMethod; 43 | import org.jenkinsci.plugins.workflow.ArtifactManagerTest; 44 | import org.jenkinsci.test.acceptance.docker.fixtures.JavaContainer; 45 | import org.junit.BeforeClass; 46 | import org.junit.ClassRule; 47 | import org.junit.Rule; 48 | import org.junit.Test; 49 | import org.jvnet.hudson.test.BuildWatcher; 50 | import org.jvnet.hudson.test.JenkinsRule; 51 | import org.jvnet.hudson.test.LoggerRule; 52 | import org.jvnet.hudson.test.TestBuilder; 53 | 54 | import com.cloudbees.hudson.plugins.folder.Folder; 55 | import com.cloudbees.plugins.credentials.CredentialsScope; 56 | import com.cloudbees.plugins.credentials.SystemCredentialsProvider; 57 | import com.cloudbees.plugins.credentials.domains.Domain; 58 | import com.cloudbees.plugins.credentials.impl.UsernamePasswordCredentialsImpl; 59 | import org.htmlunit.WebResponse; 60 | 61 | import hudson.ExtensionList; 62 | import hudson.FilePath; 63 | import hudson.Launcher; 64 | import hudson.model.AbstractBuild; 65 | import hudson.model.BuildListener; 66 | import hudson.model.FreeStyleBuild; 67 | import hudson.model.FreeStyleProject; 68 | import hudson.model.Item; 69 | import hudson.model.Run; 70 | import hudson.model.TaskListener; 71 | import hudson.plugins.sshslaves.SSHLauncher; 72 | import hudson.remoting.Which; 73 | import hudson.slaves.DumbSlave; 74 | import hudson.tasks.ArtifactArchiver; 75 | import io.jenkins.plugins.aws.global_configuration.CredentialsAwsGlobalConfiguration; 76 | import java.io.Serializable; 77 | import java.net.URL; 78 | import java.util.Collections; 79 | import java.util.Set; 80 | import jenkins.branch.BranchSource; 81 | import jenkins.model.ArtifactManagerConfiguration; 82 | import jenkins.model.ArtifactManagerFactory; 83 | import jenkins.model.Jenkins; 84 | import jenkins.plugins.git.GitSCMSource; 85 | import jenkins.plugins.git.GitSampleRepoRule; 86 | import jenkins.plugins.git.traits.BranchDiscoveryTrait; 87 | import jenkins.security.MasterToSlaveCallable; 88 | import jenkins.util.BuildListenerAdapter; 89 | import org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition; 90 | import org.jenkinsci.plugins.workflow.job.WorkflowJob; 91 | import org.jenkinsci.plugins.workflow.job.WorkflowRun; 92 | import org.jvnet.hudson.test.Issue; 93 | import org.jenkinsci.plugins.workflow.flow.FlowCopier; 94 | import org.jenkinsci.plugins.workflow.multibranch.WorkflowMultiBranchProject; 95 | import org.jenkinsci.plugins.workflow.multibranch.WorkflowMultiBranchProjectTest; 96 | import org.jenkinsci.plugins.workflow.steps.Step; 97 | import org.jenkinsci.plugins.workflow.steps.StepContext; 98 | import org.jenkinsci.plugins.workflow.steps.StepDescriptor; 99 | import org.jenkinsci.plugins.workflow.steps.StepExecution; 100 | import org.jenkinsci.plugins.workflow.steps.StepExecutions; 101 | import org.junit.AssumptionViolatedException; 102 | import org.jvnet.hudson.test.MockAuthorizationStrategy; 103 | import org.jvnet.hudson.test.TestExtension; 104 | import org.kohsuke.stapler.DataBoundConstructor; 105 | import software.amazon.awssdk.core.exception.SdkClientException; 106 | import software.amazon.awssdk.services.s3.S3Client; 107 | 108 | public class JCloudsArtifactManagerTest extends S3AbstractTest { 109 | 110 | @ClassRule 111 | public static BuildWatcher buildWatcher = new BuildWatcher(); 112 | 113 | @BeforeClass 114 | public static void live() { 115 | try { 116 | S3AbstractTest.live(); 117 | } catch (AssumptionViolatedException x) { 118 | // TODO Surefire seems to not display these at all? 119 | x.printStackTrace(); 120 | throw x; 121 | } 122 | } 123 | 124 | @Rule 125 | public LoggerRule httpLogging = new LoggerRule(); 126 | 127 | @Rule 128 | public GitSampleRepoRule sampleRepo = new GitSampleRepoRule(); 129 | 130 | protected ArtifactManagerFactory getArtifactManagerFactory(Boolean deleteArtifacts, Boolean deleteStashes) { 131 | return new JCloudsArtifactManagerFactory(new CustomBehaviorBlobStoreProvider(provider, deleteArtifacts, deleteStashes)); 132 | } 133 | 134 | @Test 135 | public void agentPermissions() throws Exception { 136 | var image = ArtifactManagerTest.prepareImage(); // TODO simplify to use Testcontainers directly 137 | assumeNotNull(image); 138 | System.err.println("verifying that while the master can connect to S3, a Dockerized agent cannot"); 139 | try (JavaContainer container = image.start(JavaContainer.class).start()) { 140 | SystemCredentialsProvider.getInstance().getDomainCredentialsMap().put(Domain.global(), Collections.singletonList(new UsernamePasswordCredentialsImpl(CredentialsScope.SYSTEM, "test", null, "test", "test"))); 141 | DumbSlave agent = new DumbSlave("assumptions", "/home/test/slave", new SSHLauncher(container.ipBound(22), container.port(22), "test")); 142 | Jenkins.get().addNode(agent); 143 | j.waitOnline(agent); 144 | try { 145 | agent.getChannel().call(new LoadS3Credentials()); 146 | fail("did not expect to be able to connect to S3 from a Dockerized agent"); // or AssumptionViolatedException? 147 | } catch (SdkClientException x) { 148 | System.err.println("a Dockerized agent was unable to connect to S3, as expected: " + x); 149 | } 150 | } 151 | } 152 | 153 | @Test 154 | public void artifactArchive() throws Exception { 155 | // To demo class loading performance: loggerRule.record(SlaveComputer.class, Level.FINEST); 156 | ArtifactManagerTest.artifactArchive(j, getArtifactManagerFactory(null, null), true, null); 157 | } 158 | 159 | @Test 160 | public void artifactArchiveAndDelete() throws Exception { 161 | ArtifactManagerTest.artifactArchiveAndDelete(j, getArtifactManagerFactory(true, null), true, null); 162 | } 163 | 164 | @Test 165 | public void artifactStash() throws Exception { 166 | ArtifactManagerTest.artifactStash(j, getArtifactManagerFactory(null, null), true, null); 167 | } 168 | 169 | @Test 170 | public void artifactStashAndDelete() throws Exception { 171 | ArtifactManagerTest.artifactStashAndDelete(j, getArtifactManagerFactory(null, true), true, null); 172 | } 173 | 174 | private static final class LoadS3Credentials extends MasterToSlaveCallable { 175 | @Override 176 | public Void call() { 177 | S3Client.builder().build(); 178 | return null; 179 | } 180 | } 181 | 182 | @Test 183 | public void artifactBrowsingPerformance() throws Exception { 184 | ArtifactManagerConfiguration.get().getArtifactManagerFactories().add(getArtifactManagerFactory(null, null)); 185 | FreeStyleProject p = j.createFreeStyleProject(); 186 | p.getBuildersList().add(new TestBuilder() { 187 | @Override 188 | public boolean perform(AbstractBuild build, Launcher launcher, BuildListener listener) throws InterruptedException, IOException { 189 | FilePath ws = build.getWorkspace(); 190 | for (int i = 0; i < 10; i++) { 191 | for (int j = 0; j < 10; j++) { 192 | ws.child(i + "/" + j + "/f").write(i + "-" + j, null); 193 | } 194 | } 195 | return true; 196 | } 197 | }); 198 | p.getPublishersList().add(new ArtifactArchiver("**")); 199 | FreeStyleBuild b = j.buildAndAssertSuccess(p); 200 | httpLogging.record(InvokeHttpMethod.class, Level.FINE); 201 | httpLogging.capture(1000); 202 | JenkinsRule.WebClient wc = j.createWebClient(); 203 | // Exercise DirectoryBrowserSupport & Run.getArtifactsUpTo 204 | System.err.println("build root"); 205 | wc.getPage(b); 206 | System.err.println("artifact root"); 207 | wc.getPage(b, "artifact/"); 208 | System.err.println("3 subdir"); 209 | wc.getPage(b, "artifact/3/"); 210 | System.err.println("3/4 subdir"); 211 | wc.getPage(b, "artifact/3/4/"); 212 | int httpCount = httpLogging.getRecords().size(); 213 | System.err.println("total count: " + httpCount); 214 | assertThat(httpCount, lessThanOrEqualTo(13)); 215 | } 216 | 217 | @Issue({"JENKINS-51390", "JCLOUDS-1200"}) 218 | @Test 219 | public void serializationProblem() throws Exception { 220 | ArtifactManagerConfiguration.get().getArtifactManagerFactories().add(getArtifactManagerFactory(null, null)); 221 | WorkflowJob p = j.createProject(WorkflowJob.class, "p"); 222 | p.setDefinition(new CpsFlowDefinition("node {writeFile file: 'f', text: 'content'; archiveArtifacts 'f'; dir('d') {try {unarchive mapping: ['f': 'f']} catch (x) {sleep 1; echo(/caught $x/)}}}", true)); 223 | S3BlobStore.BREAK_CREDS = true; 224 | try { 225 | WorkflowRun b = j.buildAndAssertSuccess(p); 226 | j.assertLogContains("caught java.io.IOException: org.jclouds.aws.AWSResponseException", b); 227 | j.assertLogNotContains("java.io.NotSerializableException", b); 228 | } finally { 229 | S3BlobStore.BREAK_CREDS = false; 230 | } 231 | } 232 | 233 | @Issue({"JENKINS-52151", "JENKINS-60040"}) 234 | @Test 235 | public void slashyBranches() throws Exception { 236 | ArtifactManagerConfiguration.get().getArtifactManagerFactories().add(getArtifactManagerFactory(true, true)); 237 | sampleRepo.init(); 238 | sampleRepo.git("checkout", "-b", "dev/main"); 239 | sampleRepo.write("Jenkinsfile", "node {dir('src') {writeFile file: 'f', text: 'content'; archiveArtifacts 'f'; stash 'x'}; dir('dest') {unstash 'x'; unarchive mapping: ['f': 'f2']}}"); 240 | sampleRepo.git("add", "Jenkinsfile"); 241 | sampleRepo.git("commit", "--message=flow"); 242 | WorkflowMultiBranchProject mp = j.jenkins.createProject(WorkflowMultiBranchProject.class, "p"); 243 | GitSCMSource gitSCMSource = new GitSCMSource(sampleRepo.toString()); 244 | gitSCMSource.setTraits(Collections.singletonList(new BranchDiscoveryTrait())); 245 | mp.getSourcesList().add(new BranchSource(gitSCMSource)); 246 | WorkflowJob p = WorkflowMultiBranchProjectTest.scheduleAndFindBranchProject(mp, "dev%2Fmain"); 247 | assertEquals(1, mp.getItems().size()); 248 | j.waitUntilNoActivity(); 249 | WorkflowRun b = p.getLastBuild(); 250 | assertEquals(1, b.getNumber()); 251 | j.assertBuildStatusSuccess(b); 252 | URL url = b.getArtifactManager().root().child("f").toExternalURL(); 253 | System.out.println("Defined: " + url); 254 | JenkinsRule.WebClient wc = j.createWebClient(); 255 | wc.getPage(b); 256 | wc.getPage(b, "artifact/"); 257 | assertEquals("content", wc.goTo(b.getUrl() + "artifact/f", null).getWebResponse().getContentAsString()); 258 | sampleRepo.write("Jenkinsfile", ""); 259 | sampleRepo.git("add", "Jenkinsfile"); 260 | sampleRepo.git("commit", "--message=empty"); 261 | WorkflowRun b2 = j.buildAndAssertSuccess(p); 262 | for (FlowCopier copier : ExtensionList.lookup(FlowCopier.class)) { 263 | copier.copy(b.asFlowExecutionOwner(), b2.asFlowExecutionOwner()); 264 | } 265 | assertTrue(b2.getArtifactManager().root().child("f").isFile()); 266 | b.deleteArtifacts(); 267 | } 268 | 269 | @Issue("JENKINS-56004") 270 | @Test 271 | public void nonAdmin() throws Exception { 272 | CredentialsAwsGlobalConfiguration.get().setCredentialsId("bogus"); // force sessionCredentials to call getCredentials 273 | ArtifactManagerConfiguration.get().getArtifactManagerFactories().add(getArtifactManagerFactory(null, null)); 274 | Folder d = j.createProject(Folder.class, "d"); 275 | j.jenkins.setSecurityRealm(j.createDummySecurityRealm()); 276 | j.jenkins.setAuthorizationStrategy(new MockAuthorizationStrategy(). 277 | grant(Jenkins.ADMINISTER).everywhere().to("admin"). 278 | grant(Jenkins.READ).everywhere().to("dev1", "dev2"). 279 | grant(Item.READ).onFolders(d).to("dev2")); 280 | WorkflowJob p = d.createProject(WorkflowJob.class, "p"); 281 | p.setDefinition(new CpsFlowDefinition("node {writeFile file: 'f.txt', text: ''; archiveArtifacts 'f.txt'}", true)); 282 | WorkflowRun b = j.buildAndAssertSuccess(p); 283 | String url = "job/d/job/p/1/api/json?tree=artifacts[relativePath]"; 284 | String jsonType = "application/json"; 285 | String snippet = "\"relativePath\":\"f.txt\""; 286 | assertThat(j.createWebClient().withBasicCredentials("admin").goTo(url, jsonType).getWebResponse().getContentAsString(), containsString(snippet)); 287 | j.createWebClient().withBasicCredentials("dev1").assertFails(url, 404); 288 | assertThat(j.createWebClient().withBasicCredentials("dev2").goTo(url, jsonType).getWebResponse().getContentAsString(), containsString(snippet)); 289 | } 290 | 291 | @Issue("JENKINS-50772") 292 | @Test 293 | public void contentType() throws Exception { 294 | String text = "some regular text"; 295 | String html = "
Test file contents"; 296 | String json = "{\"key\":\"value\"}"; 297 | ArtifactManagerConfiguration.get().getArtifactManagerFactories().add(getArtifactManagerFactory(null, null)); 298 | 299 | j.createSlave("remote", null, null); 300 | 301 | WorkflowJob p = j.createProject(WorkflowJob.class, "p"); 302 | p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f.txt', text: '" + text + "'; writeFile file: 'f.html', text: '" + html + "'; writeFile file: 'f', text: '\\u0000';writeFile file: 'f.json', text: '" + json +"'; archiveArtifacts 'f*'}", true)); 303 | j.buildAndAssertSuccess(p); 304 | 305 | WebResponse response = j.createWebClient().goTo("job/p/1/artifact/f.txt", null).getWebResponse(); 306 | assertThat(response.getContentAsString(), equalTo(text)); 307 | assertThat(response.getContentType(), equalTo("text/plain")); 308 | response = j.createWebClient().goTo("job/p/1/artifact/f.html", null).getWebResponse(); 309 | assertThat(response.getContentAsString(), equalTo(html)); 310 | assertThat(response.getContentType(), equalTo("text/html")); 311 | response = j.createWebClient().goTo("job/p/1/artifact/f", null).getWebResponse(); 312 | assertThat(response.getContentLength(), equalTo(1L)); 313 | assertThat(response.getContentType(), containsString("/octet-stream")); 314 | response = j.createWebClient().goTo("job/p/1/artifact/f.json", null).getWebResponse(); 315 | assertThat(response.getContentAsString(), equalTo(json)); 316 | assertThat(response.getContentType(), equalTo("application/json")); 317 | } 318 | 319 | @Test 320 | public void archiveWithDistinctArchiveAndWorkspacePaths() throws Exception { 321 | String text = "some regular text"; 322 | ArtifactManagerConfiguration.get().getArtifactManagerFactories().add(getArtifactManagerFactory(null, null)); 323 | 324 | j.createSlave("remote", null, null); 325 | 326 | WorkflowJob p = j.createProject(WorkflowJob.class, "p"); 327 | p.setDefinition(new CpsFlowDefinition( 328 | "node('remote') {\n" + 329 | " writeFile file: 'f.txt', text: '" + text + "'\n" + 330 | " archiveWithCustomPath(archivePath: 'what/an/interesting/path/to/f.txt', workspacePath: 'f.txt')\n" + 331 | "}", true)); 332 | j.buildAndAssertSuccess(p); 333 | 334 | WebResponse response = j.createWebClient().goTo("job/p/1/artifact/what/an/interesting/path/to/f.txt", null).getWebResponse(); 335 | assertThat(response.getContentAsString(), equalTo(text)); 336 | assertThat(response.getContentType(), equalTo("text/plain")); 337 | } 338 | 339 | //@Test 340 | public void archiveSingleLargeFile() throws Exception { 341 | ArtifactManagerConfiguration.get().getArtifactManagerFactories().add(getArtifactManagerFactory(null, null)); 342 | FreeStyleProject p = j.createFreeStyleProject(); 343 | p.getBuildersList().add(new TestBuilder() { 344 | @Override 345 | public boolean perform(AbstractBuild build, Launcher launcher, BuildListener listener) 346 | throws InterruptedException, IOException { 347 | FilePath target = build.getWorkspace().child("out"); 348 | long length = 2L * 1024 * 1024 * 1024; 349 | final FilePath src = new FilePath(Which.jarFile(Jenkins.class)); 350 | 351 | final OutputStream out = target.write(); 352 | try { 353 | do { 354 | IOUtils.copy(src.read(), out); 355 | } while (target.length() < length); 356 | } finally { 357 | out.close(); 358 | } 359 | return true; 360 | } 361 | }); 362 | p.getPublishersList().add(new ArtifactArchiver("**/*")); 363 | FreeStyleBuild build = j.buildAndAssertSuccess(p); 364 | InputStream out = build.getArtifactManager().root().child("out").open(); 365 | try { 366 | IOUtils.copy(out, new NullOutputStream()); 367 | } finally { 368 | out.close(); 369 | } 370 | } 371 | 372 | public static class ArchiveArtifactWithCustomPathStep extends Step implements Serializable { 373 | private static final long serialVersionUID = 1L; 374 | private final String archivePath; 375 | private final String workspacePath; 376 | @DataBoundConstructor 377 | public ArchiveArtifactWithCustomPathStep(String archivePath, String workspacePath) { 378 | this.archivePath = archivePath; 379 | this.workspacePath = workspacePath; 380 | } 381 | @Override 382 | public StepExecution start(StepContext context) throws Exception { 383 | return StepExecutions.synchronousNonBlocking(context, context2 -> { 384 | context.get(Run.class).pickArtifactManager().archive( 385 | context.get(FilePath.class), 386 | context.get(Launcher.class), 387 | new BuildListenerAdapter(context.get(TaskListener.class)), 388 | Collections.singletonMap(archivePath, workspacePath)); 389 | return null; 390 | }); 391 | } 392 | @TestExtension("archiveWithDistinctArchiveAndWorkspacePaths") 393 | public static class DescriptorImpl extends StepDescriptor { 394 | @Override 395 | public String getFunctionName() { 396 | return "archiveWithCustomPath"; 397 | } 398 | @Override 399 | public Set> getRequiredContext() { 400 | return Set.of(FilePath.class, Launcher.class, TaskListener.class); 401 | } 402 | } 403 | } 404 | } 405 | -------------------------------------------------------------------------------- /src/test/java/io/jenkins/plugins/artifact_manager_jclouds/s3/JCloudsVirtualFileTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2018 CloudBees, Inc. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package io.jenkins.plugins.artifact_manager_jclouds.s3; 26 | 27 | import io.jenkins.plugins.artifact_manager_jclouds.JCloudsVirtualFile; 28 | import io.jenkins.plugins.aws.global_configuration.CredentialsAwsGlobalConfiguration; 29 | import static org.hamcrest.Matchers.*; 30 | import static org.jclouds.blobstore.options.ListContainerOptions.Builder.*; 31 | import static org.junit.Assert.*; 32 | 33 | import java.io.File; 34 | import java.io.FileNotFoundException; 35 | import java.io.InputStream; 36 | import java.net.URLEncoder; 37 | import java.nio.file.Files; 38 | import java.util.Arrays; 39 | import java.util.List; 40 | import java.util.logging.Handler; 41 | import java.util.logging.Level; 42 | import java.util.logging.LogRecord; 43 | import java.util.logging.Logger; 44 | 45 | import org.apache.commons.io.FileUtils; 46 | import org.apache.commons.io.IOUtils; 47 | import org.jclouds.blobstore.domain.Blob; 48 | import org.jclouds.blobstore.domain.PageSet; 49 | import org.jclouds.blobstore.domain.StorageMetadata; 50 | import org.jclouds.rest.internal.InvokeHttpMethod; 51 | import org.junit.Rule; 52 | import org.junit.Test; 53 | import org.jvnet.hudson.test.Issue; 54 | import org.jvnet.hudson.test.LoggerRule; 55 | 56 | import java.net.ProtocolException; 57 | 58 | import jenkins.util.VirtualFile; 59 | import org.jclouds.http.HttpResponseException; 60 | import software.amazon.awssdk.services.s3.S3Client; 61 | import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; 62 | import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; 63 | import software.amazon.awssdk.services.s3.model.S3Object; 64 | 65 | public class JCloudsVirtualFileTest extends S3AbstractTest { 66 | 67 | protected static final Logger LOGGER = Logger.getLogger(JCloudsVirtualFileTest.class.getName()); 68 | 69 | protected File tmpFile; 70 | protected String filePath, missingFilePath, weirdCharactersPath; 71 | protected JCloudsVirtualFile root, subdir, vf, missing, weirdCharacters, weirdCharactersMissing; 72 | @Rule 73 | public LoggerRule httpLogging = new LoggerRule(); 74 | 75 | @Override 76 | public void setup() throws Exception { 77 | tmpFile = tmp.newFile(); 78 | Files.writeString(tmpFile.toPath(), "test"); 79 | filePath = getPrefix() + tmpFile.getName(); 80 | Blob blob = blobStore.blobBuilder(filePath).payload(tmpFile).build(); 81 | 82 | LOGGER.log(Level.INFO, "Adding test blob {0} {1}", new String[] { getContainer(), filePath }); 83 | putBlob(blob); 84 | 85 | root = newJCloudsBlobStore(S3_DIR); 86 | subdir = newJCloudsBlobStore(getPrefix()); 87 | vf = newJCloudsBlobStore(filePath); 88 | 89 | missingFilePath = getPrefix() + "missing"; 90 | missing = newJCloudsBlobStore(missingFilePath); 91 | 92 | // ampersand '&' fails the tests 93 | // it works using the aws-sdk directly so we can just assume it's a jclouds issue 94 | // https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#object-keys 95 | weirdCharactersPath = getPrefix() + "xxx#?:$'\"<>čॐ"; 96 | weirdCharacters = newJCloudsBlobStore(weirdCharactersPath); 97 | weirdCharactersMissing = newJCloudsBlobStore(weirdCharactersPath + "missing"); 98 | LOGGER.log(Level.INFO, "Adding test blob {0} {1}", new String[] { getContainer(), weirdCharactersPath }); 99 | putBlob(blobStore.blobBuilder(weirdCharactersPath).payload(tmpFile).build()); 100 | } 101 | 102 | /** Working around an apparent server flake. */ 103 | private void putBlob(Blob blob) { 104 | for (int i = 0; i < 5; i++) { 105 | try { 106 | blobStore.putBlob(getContainer(), blob); 107 | return; 108 | } catch (HttpResponseException x) { 109 | if (x.getCause() instanceof ProtocolException && i < 4) { 110 | x.printStackTrace(); 111 | continue; 112 | } 113 | throw x; 114 | } 115 | } 116 | } 117 | 118 | private JCloudsVirtualFile newJCloudsBlobStore(String path) { 119 | S3BlobStore s3BlobStore = new S3BlobStore(); 120 | return new JCloudsVirtualFile(s3BlobStore, getContainer(), path.replaceFirst("/$", "")); 121 | } 122 | 123 | @Test 124 | public void child() throws Exception { 125 | assertTrue(subdir.child(tmpFile.getName()).exists()); 126 | assertFalse(subdir.child(missing.getName()).exists()); 127 | } 128 | 129 | @Test 130 | public void exists() throws Exception { 131 | assertTrue(root.exists()); 132 | assertTrue(subdir.exists()); 133 | assertTrue(vf.exists()); 134 | assertFalse(missing.exists()); 135 | assertTrue(weirdCharacters.exists()); 136 | assertFalse(weirdCharactersMissing.exists()); 137 | } 138 | 139 | @Test 140 | public void getName() throws Exception { 141 | String[] s = getPrefix().split("/"); 142 | assertEquals(s[s.length - 1], subdir.getName()); 143 | assertEquals(tmpFile.getName(), vf.getName()); 144 | assertEquals("missing", missing.getName()); 145 | } 146 | 147 | @Test 148 | public void getParent() throws Exception { 149 | assertEquals(root, subdir.getParent().getParent()); 150 | assertEquals(subdir, vf.getParent()); 151 | assertEquals(subdir, missing.getParent()); 152 | } 153 | 154 | @Test 155 | public void isDirectory() throws Exception { 156 | assertTrue(root.isDirectory()); 157 | assertTrue(subdir.isDirectory()); 158 | assertFalse(vf.isDirectory()); 159 | assertFalse(missing.isDirectory()); 160 | assertFalse(weirdCharacters.isDirectory()); 161 | assertFalse(weirdCharactersMissing.isDirectory()); 162 | 163 | // currently fails with AuthorizationException due to ampersand see above 164 | // assertFalse(newJCloudsBlobStore(getPrefix() + "/chartest/xxx&").isDirectory()); 165 | 166 | // but this succeeds 167 | // final AmazonS3 s3 = AmazonS3ClientBuilder.defaultClient(); 168 | // ListObjectsV2Result listObjectsV2 = s3.listObjectsV2(getContainer(), getPrefix() + 169 | // "/chartest/xxx#?:$&'\"<>čॐ"); 170 | // ObjectListing listObjects = s3.listObjects(getContainer(), getPrefix() + "/chartest/xxx#?:$&'\"<>čॐ"); 171 | } 172 | 173 | @Test 174 | public void isFile() throws Exception { 175 | assertFalse(root.isFile()); 176 | assertFalse(subdir.isFile()); 177 | assertTrue(vf.isFile()); 178 | assertFalse(missing.isFile()); 179 | assertTrue(weirdCharacters.isFile()); 180 | assertFalse(weirdCharactersMissing.isFile()); 181 | } 182 | 183 | @Test 184 | public void lastModified() throws Exception { 185 | assertEquals(0, root.lastModified()); 186 | assertEquals(0, subdir.lastModified()); 187 | assertNotEquals(0, vf.lastModified()); 188 | assertEquals(0, missing.lastModified()); 189 | } 190 | 191 | @Test 192 | public void length() throws Exception { 193 | assertEquals(0, root.length()); 194 | assertEquals(0, subdir.length()); 195 | assertEquals(tmpFile.length(), vf.length()); 196 | assertEquals(0, missing.length()); 197 | } 198 | 199 | private void assertVirtualFileArrayEquals(VirtualFile[] expected, VirtualFile[] actual) { 200 | assertArrayEquals("Expected: " + Arrays.toString(expected) + " Actual: " + Arrays.toString(actual), expected, 201 | actual); 202 | } 203 | 204 | @Test 205 | public void list() throws Exception { 206 | assertVirtualFileArrayEquals(new JCloudsVirtualFile[] { vf, weirdCharacters }, subdir.list()); 207 | assertVirtualFileArrayEquals(new JCloudsVirtualFile[0], vf.list()); 208 | assertVirtualFileArrayEquals(new JCloudsVirtualFile[0], missing.list()); 209 | } 210 | 211 | @Test 212 | @SuppressWarnings("deprecation") 213 | public void listGlob() throws Exception { 214 | assertThat(subdir.list("**/**"), arrayContainingInAnyOrder(vf.getName(), weirdCharacters.getName())); 215 | assertArrayEquals(new String[] { vf.getName() }, subdir.list(tmpFile.getName().substring(0, 4) + "*")); 216 | assertArrayEquals(new String[0], subdir.list("**/something**")); 217 | assertArrayEquals(new String[0], vf.list("**/**")); 218 | assertArrayEquals(new String[0], missing.list("**/**")); 219 | } 220 | 221 | @Test 222 | public void pagedListing() throws Exception { 223 | for (int i = 0; i < 10; i++) { 224 | String iDir = getPrefix() + "sprawling/i" + i + "/"; 225 | for (int j = 0; j < 10; j++) { 226 | for (int k = 0; k < 10; k++) { 227 | putBlob(blobStore.blobBuilder(iDir + "j" + j + "/k" + k).payload(new byte[0]).build()); 228 | } 229 | } 230 | putBlob(blobStore.blobBuilder(iDir + "extra").payload(new byte[0]).build()); 231 | LOGGER.log(Level.INFO, "added 101 blobs to {0}", iDir); 232 | } 233 | httpLogging.record(InvokeHttpMethod.class, Level.FINE); 234 | httpLogging.capture(1000); 235 | Logger.getLogger(InvokeHttpMethod.class.getName()).addHandler(new Handler() { 236 | {setLevel(Level.FINE);} 237 | int count; 238 | @Override public void publish(LogRecord record) { 239 | if (record.getMessage().contains("invoking GetBucketLocation")) { 240 | new Exception("calling GetBucketLocation #" + count++).printStackTrace(); 241 | if (count > 1) { 242 | throw new IllegalStateException("should only ever have to call GetBucketLocation once"); 243 | } 244 | } 245 | } 246 | @Override public void flush() {} 247 | @Override public void close() throws SecurityException {} 248 | }); 249 | // Default list page size for S3 is 1000 blobs; we have 1010 plus the two created for all tests, so should hit a second page. 250 | assertThat(subdir.list("sprawling/**/k3", null, true), iterableWithSize(100)); 251 | assertEquals("calls GetBucketLocation then ListBucket, advance to …/sprawling/i9/j8/k8, ListBucket again", 3, httpLogging.getRecords().size()); 252 | } 253 | 254 | @Test 255 | public void open() throws Exception { 256 | try (InputStream is = subdir.open()) { 257 | fail("Should not open a dir"); 258 | } catch (FileNotFoundException e) { 259 | // expected 260 | } 261 | try (InputStream is = missing.open()) { 262 | fail("Should not open a missing file"); 263 | } catch (FileNotFoundException e) { 264 | // expected 265 | } 266 | try (InputStream is = vf.open()) { 267 | assertEquals(FileUtils.readFileToString(tmpFile), IOUtils.toString(is)); 268 | } 269 | } 270 | 271 | @Test 272 | public void toURI() throws Exception { 273 | assertEquals(String.format("https://%s.s3.amazonaws.com/%s", getContainer(), urlEncodeParts(getPrefix().replaceFirst("/$", ""))), subdir.toURI().toString()); 274 | assertEquals(String.format("https://%s.s3.amazonaws.com/%s", getContainer(), urlEncodeParts(filePath)), vf.toURI().toString()); 275 | // weird chars 276 | String stuff = "xxx#?:$&'\"<>čॐ"; 277 | assertEquals(String.format("https://%s.s3.amazonaws.com/%s", getContainer(), urlEncodeParts(stuff)), newJCloudsBlobStore(stuff).toURI().toString()); 278 | // region 279 | CredentialsAwsGlobalConfiguration.get().setRegion("us-west-1"); 280 | assertEquals(String.format("https://%s.s3.us-west-1.amazonaws.com/what/ever", getContainer()), newJCloudsBlobStore("what/ever").toURI().toString()); 281 | } 282 | private static String urlEncodeParts(String s) throws Exception { 283 | return URLEncoder.encode(s, "UTF-8").replaceAll("%2F", "/"); 284 | } 285 | 286 | @Test 287 | @Issue({ "JENKINS-50591", "JCLOUDS-1401" }) 288 | public void testAmpersand() throws Exception { 289 | String key = getPrefix() + "xxx#?:&$'\"<>čॐ"; 290 | 291 | try { 292 | putBlob(blobStore.blobBuilder(key).payload("test").build()); 293 | 294 | final S3Client s3 = S3Client.create(); 295 | ListObjectsV2Response result = s3.listObjectsV2(ListObjectsV2Request.builder().bucket(getContainer()).build()); 296 | List objects = result.contents(); 297 | assertThat(objects, not(empty())); 298 | 299 | // fails with 300 | // org.jclouds.rest.AuthorizationException: The request signature we calculated does not match the signature 301 | // you provided. Check your key and signing method. 302 | PageSet list = blobStore.list(getContainer(), prefix(key)); 303 | assertThat(list, not(empty())); 304 | } finally { 305 | blobStore.removeBlob(getContainer(), key); 306 | } 307 | } 308 | } 309 | -------------------------------------------------------------------------------- /src/test/java/io/jenkins/plugins/artifact_manager_jclouds/s3/LocalStackIntegrationTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2018 CloudBees, Inc. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package io.jenkins.plugins.artifact_manager_jclouds.s3; 25 | 26 | import com.cloudbees.jenkins.plugins.awscredentials.AWSCredentialsImpl; 27 | import com.cloudbees.plugins.credentials.CredentialsProvider; 28 | import com.cloudbees.plugins.credentials.CredentialsScope; 29 | import com.cloudbees.plugins.credentials.domains.Domain; 30 | import io.jenkins.plugins.aws.global_configuration.CredentialsAwsGlobalConfiguration; 31 | import jenkins.model.Jenkins; 32 | import org.apache.commons.lang.StringUtils; 33 | import org.jclouds.aws.domain.Region; 34 | import org.junit.AfterClass; 35 | import org.junit.BeforeClass; 36 | import org.testcontainers.Testcontainers; 37 | import org.testcontainers.containers.localstack.LocalStackContainer; 38 | import org.testcontainers.utility.DockerImageName; 39 | 40 | import java.util.Locale; 41 | 42 | import org.junit.Before; 43 | import static org.testcontainers.containers.localstack.LocalStackContainer.Service.S3; 44 | 45 | public class LocalStackIntegrationTest extends AbstractIntegrationTest { 46 | private static LocalStackContainer LOCALSTACK; 47 | 48 | 49 | @BeforeClass 50 | public static void setUpClass() throws Exception { 51 | LOCALSTACK = new LocalStackContainer(DockerImageName.parse("localstack/localstack:4.4.0")) 52 | .withServices(S3); 53 | LOCALSTACK.start(); 54 | Integer mappedPort = LOCALSTACK.getFirstMappedPort(); 55 | Testcontainers.exposeHostPorts(mappedPort); 56 | } 57 | 58 | @AfterClass 59 | public static void shutDownClass() { 60 | if (LOCALSTACK != null && LOCALSTACK.isRunning()) { 61 | LOCALSTACK.stop(); 62 | } 63 | } 64 | 65 | @Before public void configure() throws Throwable { 66 | rr.startJenkins(); 67 | var endpoint = LOCALSTACK.getEndpoint().getHost() + ":" + LOCALSTACK.getEndpoint().getPort(); 68 | var username = LOCALSTACK.getAccessKey(); 69 | var password = LOCALSTACK.getSecretKey(); 70 | var region = LOCALSTACK.getRegion(); 71 | rr.run(r -> { 72 | CredentialsAwsGlobalConfiguration credentialsConfig = CredentialsAwsGlobalConfiguration.get(); 73 | credentialsConfig.setRegion(region); 74 | CredentialsProvider.lookupStores(Jenkins.get()) 75 | .iterator() 76 | .next() 77 | .addCredentials(Domain.global(), new AWSCredentialsImpl(CredentialsScope.GLOBAL, "LocalStackIntegrationTest", username, password, null)); 78 | credentialsConfig.setCredentialsId("LocalStackIntegrationTest"); 79 | 80 | var config = S3BlobStoreConfig.get(); 81 | config.setContainer(CONTAINER_NAME); 82 | config.setPrefix(CONTAINER_PREFIX); 83 | config.setCustomEndpoint(endpoint); 84 | config.setUseHttp(true); 85 | config.setUsePathStyleUrl(true); 86 | config.setDisableSessionToken(true); 87 | config.setCustomSigningRegion(StringUtils.isBlank(region) ? Region.US_EAST_1.toLowerCase(Locale.US) : region); 88 | }); 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /src/test/java/io/jenkins/plugins/artifact_manager_jclouds/s3/MinioIntegrationTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2018 CloudBees, Inc. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package io.jenkins.plugins.artifact_manager_jclouds.s3; 25 | 26 | import com.cloudbees.jenkins.plugins.awscredentials.AWSCredentialsImpl; 27 | import com.cloudbees.plugins.credentials.CredentialsProvider; 28 | import com.cloudbees.plugins.credentials.CredentialsScope; 29 | import com.cloudbees.plugins.credentials.domains.Domain; 30 | import io.jenkins.plugins.aws.global_configuration.CredentialsAwsGlobalConfiguration; 31 | import jenkins.model.Jenkins; 32 | 33 | import org.junit.AfterClass; 34 | import org.junit.BeforeClass; 35 | import org.testcontainers.containers.MinIOContainer; 36 | import org.junit.Before; 37 | 38 | public class MinioIntegrationTest extends AbstractIntegrationTest { 39 | private static final String REGION = "us-east-1"; 40 | 41 | private static MinIOContainer minioServer; 42 | 43 | @BeforeClass 44 | public static void setUpClass() throws Exception { 45 | minioServer = new MinIOContainer("minio/minio"); 46 | minioServer.start(); 47 | } 48 | 49 | @AfterClass 50 | public static void shutDownClass() { 51 | if (minioServer != null && minioServer.isRunning()) { 52 | minioServer.stop(); 53 | } 54 | } 55 | 56 | @Before public void configure() throws Throwable { 57 | rr.startJenkins(); 58 | var endpoint = minioServer.getS3URL().replaceFirst("^http://", ""); 59 | var username = minioServer.getUserName(); 60 | var password = minioServer.getPassword(); 61 | rr.run(r -> { 62 | CredentialsAwsGlobalConfiguration credentialsConfig = CredentialsAwsGlobalConfiguration.get(); 63 | credentialsConfig.setRegion(REGION); 64 | CredentialsProvider.lookupStores(Jenkins.get()) 65 | .iterator() 66 | .next() 67 | .addCredentials(Domain.global(), new AWSCredentialsImpl(CredentialsScope.GLOBAL, "MinioIntegrationTest", username, password, null)); 68 | credentialsConfig.setCredentialsId("MinioIntegrationTest"); 69 | 70 | var config = S3BlobStoreConfig.get(); 71 | config.setContainer(CONTAINER_NAME); 72 | config.setPrefix(CONTAINER_PREFIX); 73 | config.setCustomEndpoint(endpoint); 74 | config.setCustomSigningRegion(REGION); 75 | config.setUseHttp(true); 76 | config.setUsePathStyleUrl(true); 77 | config.setDisableSessionToken(true); 78 | }); 79 | } 80 | 81 | } 82 | -------------------------------------------------------------------------------- /src/test/java/io/jenkins/plugins/artifact_manager_jclouds/s3/S3AbstractTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2018 CloudBees, Inc. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package io.jenkins.plugins.artifact_manager_jclouds.s3; 26 | 27 | import static org.hamcrest.Matchers.is; 28 | import static org.hamcrest.Matchers.notNullValue; 29 | import static org.junit.Assume.assumeNoException; 30 | import static org.junit.Assume.assumeThat; 31 | 32 | import java.time.ZonedDateTime; 33 | import java.time.format.DateTimeFormatter; 34 | import java.util.logging.Level; 35 | 36 | import org.apache.commons.lang.RandomStringUtils; 37 | import org.jclouds.blobstore.BlobStore; 38 | import org.jclouds.blobstore.BlobStoreContext; 39 | import org.junit.After; 40 | import org.junit.Before; 41 | import org.junit.BeforeClass; 42 | import org.junit.Rule; 43 | import org.junit.rules.TemporaryFolder; 44 | import org.jvnet.hudson.test.JenkinsRule; 45 | import org.jvnet.hudson.test.LoggerRule; 46 | 47 | import io.jenkins.plugins.artifact_manager_jclouds.JCloudsVirtualFile; 48 | import io.jenkins.plugins.aws.global_configuration.CredentialsAwsGlobalConfiguration; 49 | import software.amazon.awssdk.core.exception.SdkClientException; 50 | import software.amazon.awssdk.services.s3.S3Client; 51 | import software.amazon.awssdk.services.s3.model.HeadBucketRequest; 52 | 53 | public abstract class S3AbstractTest { 54 | private static final String S3_BUCKET = System.getenv("S3_BUCKET"); 55 | protected static final String S3_DIR = System.getenv("S3_DIR"); 56 | private static final String S3_REGION = System.getenv("S3_REGION"); 57 | 58 | protected S3BlobStore provider; 59 | 60 | @BeforeClass 61 | public static void live() { 62 | assumeThat("define $S3_BUCKET as explained in README", S3_BUCKET, notNullValue()); 63 | assumeThat("define $S3_DIR as explained in README", S3_DIR, notNullValue()); 64 | 65 | try (S3Client client = S3Client.create()) { 66 | assumeThat(client.headBucket(HeadBucketRequest.builder().bucket(S3_BUCKET).build()).sdkHttpResponse().isSuccessful(), is(true)); 67 | } catch (SdkClientException x) { 68 | x.printStackTrace(); 69 | assumeNoException("failed to connect to S3 with current credentials", x); 70 | } 71 | } 72 | 73 | @Rule 74 | public TemporaryFolder tmp = new TemporaryFolder(); 75 | 76 | @Rule 77 | public LoggerRule loggerRule = new LoggerRule(); 78 | 79 | @Rule 80 | public JenkinsRule j = new JenkinsRule(); 81 | 82 | protected BlobStoreContext context; 83 | protected BlobStore blobStore; 84 | private String prefix; 85 | 86 | public static String getContainer() { 87 | return S3_BUCKET; 88 | } 89 | 90 | /** 91 | * To run each test in its own subdir 92 | */ 93 | public static String generateUniquePrefix() { 94 | return String.format("%s%s-%s/", S3_DIR, ZonedDateTime.now().format(DateTimeFormatter.ISO_INSTANT), 95 | RandomStringUtils.randomAlphabetic(4).toLowerCase()); 96 | } 97 | 98 | protected String getPrefix() { 99 | return prefix; 100 | } 101 | 102 | @Before 103 | public void setupContext() throws Exception { 104 | 105 | provider = new S3BlobStore(); 106 | S3BlobStoreConfig config = S3BlobStoreConfig.get(); 107 | config.setContainer(S3_BUCKET); 108 | 109 | CredentialsAwsGlobalConfiguration credentialsConfig = CredentialsAwsGlobalConfiguration.get(); 110 | credentialsConfig.setRegion(S3_REGION); 111 | 112 | loggerRule.recordPackage(JCloudsVirtualFile.class, Level.FINE); 113 | 114 | // run each test under its own dir 115 | prefix = generateUniquePrefix(); 116 | config.setPrefix(prefix); 117 | 118 | context = provider.getContext(); 119 | 120 | blobStore = context.getBlobStore(); 121 | 122 | setup(); 123 | } 124 | 125 | public void setup() throws Exception { 126 | } 127 | 128 | @After 129 | public void tearDown() throws Exception { 130 | if (context != null) { 131 | context.close(); 132 | } 133 | } 134 | 135 | @After 136 | public void deleteBlobs() throws Exception { 137 | JCloudsVirtualFile.delete(provider, blobStore, prefix); 138 | } 139 | 140 | } 141 | -------------------------------------------------------------------------------- /src/test/java/io/jenkins/plugins/artifact_manager_jclouds/s3/S3BlobStoreConfigFipsEnabledTest.java: -------------------------------------------------------------------------------- 1 | package io.jenkins.plugins.artifact_manager_jclouds.s3; 2 | 3 | import org.junit.Rule; 4 | import org.junit.Test; 5 | import org.jvnet.hudson.test.JenkinsRule; 6 | import org.jvnet.hudson.test.RealJenkinsRule; 7 | import hudson.util.FormValidation; 8 | import static org.junit.Assert.assertEquals; 9 | 10 | import java.io.IOException; 11 | 12 | 13 | public class S3BlobStoreConfigFipsEnabledTest { 14 | 15 | @Rule 16 | public RealJenkinsRule rule = new RealJenkinsRule().omitPlugins("eddsa-api").javaOptions("-Djenkins.security.FIPS140.COMPLIANCE=true"); 17 | 18 | 19 | @Test 20 | public void checkUseHttpsWithFipsEnabledTest() throws Throwable { 21 | rule.then(S3BlobStoreConfigFipsEnabledTest::checkUseHttpsWithFipsEnabled); 22 | } 23 | 24 | 25 | private static void checkUseHttpsWithFipsEnabled(JenkinsRule r) throws IOException { 26 | S3BlobStoreConfig descriptor = S3BlobStoreConfig.get(); 27 | assertEquals(descriptor.doCheckUseHttp(true).kind , FormValidation.Kind.ERROR); 28 | assertEquals(descriptor.doCheckUseHttp(false).kind , FormValidation.Kind.OK); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/test/java/io/jenkins/plugins/artifact_manager_jclouds/s3/S3BlobStoreConfigTest.java: -------------------------------------------------------------------------------- 1 | package io.jenkins.plugins.artifact_manager_jclouds.s3; 2 | 3 | import java.util.logging.Logger; 4 | 5 | import org.junit.Rule; 6 | import org.junit.Test; 7 | import org.jvnet.hudson.test.JenkinsRule; 8 | import io.jenkins.plugins.artifact_manager_jclouds.BlobStoreProvider; 9 | import io.jenkins.plugins.artifact_manager_jclouds.JCloudsArtifactManagerFactory; 10 | 11 | import hudson.model.Failure; 12 | import hudson.util.FormValidation; 13 | import jenkins.model.ArtifactManagerConfiguration; 14 | 15 | import static org.hamcrest.MatcherAssert.assertThat; 16 | import static org.hamcrest.Matchers.instanceOf; 17 | import static org.junit.Assert.assertEquals; 18 | import static org.junit.Assert.assertTrue; 19 | import static org.junit.Assert.fail; 20 | 21 | public class S3BlobStoreConfigTest { 22 | 23 | private static final Logger LOGGER = Logger.getLogger(S3BlobStoreConfigTest.class.getName()); 24 | 25 | public static final String CONTAINER_NAME = "container-name"; 26 | public static final String CONTAINER_PREFIX = "container-prefix/"; 27 | public static final String CONTAINER_REGION = "us-west-1"; 28 | public static final String CUSTOM_ENDPOINT = "internal-s3.company.org:9000"; 29 | public static final String CUSTOM_ENDPOINT_SIGNING_REGION = "us-west-2"; 30 | public static final boolean USE_PATH_STYLE = true; 31 | public static final boolean USE_HTTP = true; 32 | public static final boolean DISABLE_SESSION_TOKEN = true; 33 | 34 | @Rule 35 | public JenkinsRule j = new JenkinsRule(); 36 | 37 | @Test 38 | public void checkConfigurationManually() throws Exception { 39 | S3BlobStore provider = new S3BlobStore(); 40 | S3BlobStoreConfig config = S3BlobStoreConfig.get(); 41 | config.setContainer(CONTAINER_NAME); 42 | config.setPrefix(CONTAINER_PREFIX); 43 | config.setCustomEndpoint(CUSTOM_ENDPOINT); 44 | config.setCustomSigningRegion(CUSTOM_ENDPOINT_SIGNING_REGION); 45 | config.setUsePathStyleUrl(USE_PATH_STYLE); 46 | config.setUseHttp(USE_HTTP); 47 | config.setDisableSessionToken(DISABLE_SESSION_TOKEN); 48 | 49 | JCloudsArtifactManagerFactory artifactManagerFactory = new JCloudsArtifactManagerFactory(provider); 50 | ArtifactManagerConfiguration.get().getArtifactManagerFactories().add(artifactManagerFactory); 51 | 52 | LOGGER.info(artifactManagerFactory.getProvider().toString()); 53 | BlobStoreProvider providerConfigured = artifactManagerFactory.getProvider(); 54 | assertThat(providerConfigured, instanceOf(S3BlobStore.class)); 55 | checkFieldValues(config); 56 | 57 | //check configuration page submit 58 | j.configRoundtrip(); 59 | checkFieldValues(config); 60 | } 61 | 62 | private void checkFieldValues(S3BlobStoreConfig configuration) { 63 | assertEquals(CONTAINER_NAME, configuration.getContainer()); 64 | assertEquals(CONTAINER_PREFIX, configuration.getPrefix()); 65 | assertEquals(CUSTOM_ENDPOINT, S3BlobStoreConfig.get().getCustomEndpoint()); 66 | assertEquals(CUSTOM_ENDPOINT_SIGNING_REGION, S3BlobStoreConfig.get().getCustomSigningRegion()); 67 | assertEquals(USE_PATH_STYLE, S3BlobStoreConfig.get().getUsePathStyleUrl()); 68 | assertEquals(USE_HTTP, S3BlobStoreConfig.get().getUseHttp()); 69 | assertEquals(DISABLE_SESSION_TOKEN, S3BlobStoreConfig.get().getDisableSessionToken()); 70 | } 71 | 72 | @Test(expected = Failure.class) 73 | public void checkContainerWrongConfiguration() { 74 | S3BlobStoreConfig descriptor = S3BlobStoreConfig.get(); 75 | descriptor.setContainer("/wrong-container-name"); 76 | fail(); 77 | } 78 | 79 | @Test 80 | public void checkValidationsContainer() { 81 | S3BlobStoreConfig descriptor = S3BlobStoreConfig.get(); 82 | assertEquals(descriptor.doCheckContainer("aaa").kind, FormValidation.Kind.OK); 83 | assertEquals(descriptor.doCheckContainer("aaa12345678901234567890123456789012345678901234568901234567890") 84 | .kind, FormValidation.Kind.OK); 85 | assertEquals(descriptor.doCheckContainer("name.1name.name1").kind, FormValidation.Kind.OK); 86 | assertEquals(descriptor.doCheckContainer("name-1name-name1").kind, FormValidation.Kind.OK); 87 | 88 | assertEquals(descriptor.doCheckContainer("AAA").kind, FormValidation.Kind.ERROR); 89 | assertEquals(descriptor.doCheckContainer("A_A").kind, FormValidation.Kind.ERROR); 90 | assertEquals(descriptor.doCheckContainer("Name").kind, FormValidation.Kind.ERROR); 91 | assertEquals(descriptor.doCheckContainer("-name").kind, FormValidation.Kind.ERROR); 92 | assertEquals(descriptor.doCheckContainer(".name").kind, FormValidation.Kind.ERROR); 93 | assertEquals(descriptor.doCheckContainer("_name").kind, FormValidation.Kind.ERROR); 94 | assertEquals(descriptor.doCheckContainer("192.168.1.100").kind, FormValidation.Kind.ERROR); 95 | assertEquals(descriptor.doCheckContainer("name-Name").kind, FormValidation.Kind.ERROR); 96 | assertEquals(descriptor.doCheckContainer("name-namE").kind, FormValidation.Kind.ERROR); 97 | assertEquals(descriptor.doCheckContainer("name_").kind, FormValidation.Kind.ERROR); 98 | assertEquals(descriptor.doCheckContainer("/name").kind, FormValidation.Kind.ERROR); 99 | } 100 | 101 | @Test 102 | public void checkValidationsPrefix() { 103 | S3BlobStoreConfig descriptor = S3BlobStoreConfig.get(); 104 | assertEquals(descriptor.doCheckPrefix("").kind, FormValidation.Kind.OK); 105 | assertEquals(descriptor.doCheckPrefix("folder/").kind, FormValidation.Kind.OK); 106 | assertEquals(descriptor.doCheckPrefix("folder").kind, FormValidation.Kind.ERROR); 107 | } 108 | 109 | @Test 110 | public void checkValidationCustomEndPoint() { 111 | S3BlobStoreConfig descriptor = S3BlobStoreConfig.get(); 112 | assertEquals(descriptor.doCheckCustomEndpoint("").kind, FormValidation.Kind.OK); 113 | assertEquals(descriptor.doCheckCustomEndpoint("server").kind, FormValidation.Kind.OK); 114 | assertEquals(descriptor.doCheckCustomEndpoint("server.organisation.tld").kind, FormValidation.Kind.OK); 115 | assertEquals(descriptor.doCheckCustomEndpoint("server:8080").kind, FormValidation.Kind.OK); 116 | assertEquals(descriptor.doCheckCustomEndpoint("server.organisation.tld:8080").kind, FormValidation.Kind.OK); 117 | assertEquals(descriptor.doCheckCustomEndpoint("s3-server.organisation.tld").kind, FormValidation.Kind.OK); 118 | assertEquals(descriptor.doCheckCustomEndpoint("-server.organisation.tld").kind, FormValidation.Kind.ERROR); 119 | assertEquals(descriptor.doCheckCustomEndpoint(".server.organisation.tld").kind, FormValidation.Kind.ERROR); 120 | } 121 | 122 | @Test 123 | public void checkValidationCustomSigningRegion() { 124 | S3BlobStoreConfig descriptor = S3BlobStoreConfig.get(); 125 | assertEquals(descriptor.doCheckCustomSigningRegion("anystring").kind, FormValidation.Kind.OK); 126 | assertEquals(descriptor.doCheckCustomSigningRegion("").kind, FormValidation.Kind.OK); 127 | descriptor.setCustomEndpoint("server"); 128 | assertTrue(descriptor.doCheckCustomSigningRegion("").getMessage().contains("us-east-1")); 129 | } 130 | 131 | @Test 132 | public void checkValidationUseHttpsWithFipsDisabled() { 133 | S3BlobStoreConfig descriptor = S3BlobStoreConfig.get(); 134 | assertEquals(descriptor.doCheckUseHttp(true).kind , FormValidation.Kind.OK); 135 | assertEquals(descriptor.doCheckUseHttp(false).kind , FormValidation.Kind.OK); 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /src/test/resources/io/jenkins/plugins/artifact_manager_jclouds/s3/configuration-as-code.yml: -------------------------------------------------------------------------------- 1 | --- 2 | unclassified: 3 | artifactManager: 4 | artifactManagerFactories: 5 | - jclouds: 6 | provider: s3 7 | aws: 8 | awsCredentials: 9 | region: "us-east-1" 10 | s3: 11 | container: "${ARTIFACT_MANAGER_S3_BUCKET_NAME}" 12 | prefix: "jenkins_data/" 13 | customEndpoint: "internal-s3.company.org" 14 | customSigningRegion: "us-west-2" 15 | usePathStyleUrl: true 16 | useHttp: true 17 | disableSessionToken: true 18 | --------------------------------------------------------------------------------