├── gradle.properties ├── s3-artifact-storage-common ├── src │ ├── test │ │ ├── resources │ │ │ └── S3UtilsTest │ │ │ │ ├── file.txt │ │ │ │ ├── file.css │ │ │ │ ├── file.htm │ │ │ │ ├── file.html │ │ │ │ ├── file.bin │ │ │ │ ├── file.jpg │ │ │ │ └── file.zip │ │ └── java │ │ │ ├── testng.xml │ │ │ └── jetbrains │ │ │ └── buildServer │ │ │ └── artifacts │ │ │ └── s3 │ │ │ └── S3UtilTest.java │ └── main │ │ └── java │ │ └── jetbrains │ │ └── buildServer │ │ └── artifacts │ │ └── s3 │ │ └── S3ArtifactUtil.java └── build.gradle ├── s3-artifact-storage-ui ├── .eslintignore ├── .gitignore ├── .prettierrc ├── .npmrc ├── .babelrc ├── src │ ├── types │ │ ├── declarations.d.ts │ │ └── global.d.ts │ ├── index.tsx │ ├── App │ │ ├── MultipartUpload │ │ │ ├── components │ │ │ │ ├── CustomizeUpload.tsx │ │ │ │ ├── PartSize.tsx │ │ │ │ └── Threshold.tsx │ │ │ └── MultipartUploadSection.tsx │ │ ├── S3 │ │ │ ├── components │ │ │ │ ├── IAMRole.tsx │ │ │ │ └── DefaultProviderChain.tsx │ │ │ └── TransferSpeedUp │ │ │ │ └── components │ │ │ │ ├── UploadDistribution.tsx │ │ │ │ ├── DownloadDistribution.tsx │ │ │ │ └── Distribution.tsx │ │ ├── components │ │ │ ├── BucketPrefix.tsx │ │ │ ├── Bucket.tsx │ │ │ └── StorageTypeChangedWarningDialog.tsx │ │ ├── Storage │ │ │ ├── StorageSection.tsx │ │ │ └── components │ │ │ │ ├── StorageId.tsx │ │ │ │ ├── StorageName.tsx │ │ │ │ └── StorageType.tsx │ │ ├── S3Compatible │ │ │ ├── S3Section.tsx │ │ │ └── components │ │ │ │ ├── AccessKeyId.tsx │ │ │ │ ├── SecretAccessKey.tsx │ │ │ │ └── Endpoint.tsx │ │ ├── ProtocolSettings │ │ │ └── ProtocolSettings.tsx │ │ └── styles.css │ ├── hooks │ │ ├── useStorageOptions.tsx │ │ ├── useCanLoadBucketInfoData.tsx │ │ ├── useBucketOptions.tsx │ │ ├── useBucketLocation.tsx │ │ └── useTransferAccelerationAvailable.tsx │ ├── Utilities │ │ ├── fetchHelper.ts │ │ ├── fetchBucketNames.ts │ │ ├── fetchBucketLocation.ts │ │ ├── fetchS3TransferAccelerationAvailability.ts │ │ ├── fetchPublicKeys.ts │ │ ├── responseParser.ts │ │ ├── fetchCfKeysValidationResult.ts │ │ └── fetchDistributions.ts │ └── contexts │ │ └── AppContext.tsx ├── third-party-licenses-json.js ├── tsconfig.json └── webpack.config.js ├── s3-artifact-storage-server ├── src │ ├── main │ │ ├── resources │ │ │ ├── buildServerResources │ │ │ │ ├── .gitignore │ │ │ │ └── s3_storage_settings.jsp │ │ │ └── META-INF │ │ │ │ └── build-server-plugin-s3-storage.xml │ │ └── java │ │ │ └── jetbrains │ │ │ └── buildServer │ │ │ ├── artifacts │ │ │ └── s3 │ │ │ │ ├── cleanup │ │ │ │ ├── CleanupListener.java │ │ │ │ └── AbstractCleanupListener.java │ │ │ │ ├── S3CompatibleArtifactContentProvider.java │ │ │ │ ├── orphans │ │ │ │ ├── OrphanedArtifacts.java │ │ │ │ ├── BuildEntry.java │ │ │ │ ├── BuildTypeEntry.java │ │ │ │ ├── OrphanedArtifact.java │ │ │ │ └── ProjectEntry.java │ │ │ │ ├── web │ │ │ │ └── S3CompatibleArtifactDownloadProcessor.java │ │ │ │ ├── util │ │ │ │ ├── ParamUtil.java │ │ │ │ └── RegionSortPriority.java │ │ │ │ └── settings │ │ │ │ ├── S3PropertiesProcessor.java │ │ │ │ └── CloudFrontPropertiesProcessor.java │ │ │ └── filestorage │ │ │ └── cloudfront │ │ │ └── GuardedCloudFrontPresignedUrlProvider.java │ └── test │ │ └── java │ │ └── testng-docker-testcontainers.xml ├── teamcity-plugin.xml ├── build.gradle └── kotlin-dsl │ ├── S3CommonSettings.xml │ └── S3CompatibleStorageSettings.xml ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── teamcity-s3-sdk ├── src │ ├── main │ │ └── java │ │ │ └── jetbrains │ │ │ └── buildServer │ │ │ └── artifacts │ │ │ └── s3 │ │ │ ├── S3Dto.java │ │ │ ├── publish │ │ │ ├── presigned │ │ │ │ ├── upload │ │ │ │ │ ├── PublishingInterruptedException.java │ │ │ │ │ ├── PresignedUrlsProviderClientFactory.java │ │ │ │ │ ├── TeamCityServerPresignedUrlsProviderClientFactory.java │ │ │ │ │ ├── PresignedUploadProgressListener.java │ │ │ │ │ └── PresignedUrlsProviderClient.java │ │ │ │ └── util │ │ │ │ │ ├── Throwables.java │ │ │ │ │ ├── CloseableS3SignedUrlUploadPool.java │ │ │ │ │ ├── RepeatableFilePartRequestEntityApacheLegacy.java │ │ │ │ │ ├── S3MultipartUploadFileSplitter.java │ │ │ │ │ ├── RepeatableFilePartRequestEntityApache43.java │ │ │ │ │ ├── DigestUtil.java │ │ │ │ │ └── S3ErrorDto.java │ │ │ ├── logger │ │ │ │ ├── S3UploadLogger.java │ │ │ │ ├── S3Log4jUploadLogger.java │ │ │ │ └── CompositeS3UploadLogger.java │ │ │ ├── errors │ │ │ │ ├── ResponseAdapter.java │ │ │ │ ├── HttpResponseErrorHandler.java │ │ │ │ ├── HttpResponseAdapter.java │ │ │ │ ├── TeamCityPresignedUrlsProviderErrorHandler.java │ │ │ │ ├── S3ServerResponseErrorHandler.java │ │ │ │ └── S3DirectResponseErrorHandler.java │ │ │ ├── S3FileUploaderFactory.java │ │ │ └── S3FileUploaderFactoryImpl.java │ │ │ ├── amazonClient │ │ │ ├── WithS3Client.java │ │ │ ├── WithS3Presigner.java │ │ │ ├── WithCloudFrontClient.java │ │ │ └── AmazonS3Provider.java │ │ │ ├── cloudfront │ │ │ ├── CloudFrontUtils.java │ │ │ ├── RequestMetadata.java │ │ │ ├── CloudFrontPresignedUrlProvider.java │ │ │ ├── CloudFrontSettings.java │ │ │ ├── CloudFrontEnabledPresignedUrlProvider.java │ │ │ └── CloudFrontConstants.java │ │ │ ├── serialization │ │ │ ├── XmlSerializer.java │ │ │ ├── impl │ │ │ │ ├── JDomElementDeserializer.java │ │ │ │ └── XmlSerializerImpl.java │ │ │ └── S3XmlSerializerFactory.java │ │ │ ├── PresignedUrlWithTtl.java │ │ │ ├── S3Settings.java │ │ │ ├── exceptions │ │ │ ├── InvalidSettingsException.java │ │ │ └── FileUploadFailedException.java │ │ │ ├── transport │ │ │ ├── MultipartUploadStartRequestDto.java │ │ │ ├── PresignedUrlPartDto.java │ │ │ ├── MultipartUploadAbortRequestDto.java │ │ │ ├── MultipartUploadCompleteRequestDto.java │ │ │ ├── PresignedUrlListResponseDto.java │ │ │ ├── AmazonServiceErrorDto.java │ │ │ └── PresignedUrlDto.java │ │ │ ├── FileUploadInfo.java │ │ │ ├── SSLParamUtil.java │ │ │ ├── S3ClientResourceFetcher.java │ │ │ ├── S3PresignedUrlProvider.java │ │ │ ├── S3Configuration.java │ │ │ └── CheckS3TransferAccelerationAvailability.java │ └── test │ │ └── java │ │ ├── testng.xml │ │ └── jetbrains │ │ └── buildServer │ │ └── artifacts │ │ └── s3 │ │ └── S3DtoSerializerTest.java └── build.gradle ├── s3-artifact-storage-agent ├── src │ ├── test │ │ ├── resources │ │ │ └── artifacts │ │ │ │ └── file.zip │ │ └── java │ │ │ ├── testng.xml │ │ │ └── jetbrains │ │ │ └── buildServer │ │ │ └── artifacts │ │ │ └── s3 │ │ │ ├── publish │ │ │ └── presigned │ │ │ │ └── util │ │ │ │ └── S3MultipartUploadFileSplitterTest.java │ │ │ └── download │ │ │ └── S3MockContainer.java │ └── main │ │ ├── java │ │ └── jetbrains │ │ │ └── buildServer │ │ │ └── artifacts │ │ │ └── s3 │ │ │ ├── download │ │ │ ├── IORunnable.java │ │ │ ├── parallel │ │ │ │ ├── splitter │ │ │ │ │ ├── FileSplitter.java │ │ │ │ │ └── SplitabilityReport.java │ │ │ │ ├── PartFailure.java │ │ │ │ ├── strategy │ │ │ │ │ └── ParallelDownloadStrategy.java │ │ │ │ ├── FilePart.java │ │ │ │ ├── ParallelDownloadState.java │ │ │ │ └── ParallelDownloadContext.java │ │ │ └── S3HttpClient.java │ │ │ ├── publish │ │ │ ├── logger │ │ │ │ └── BuildLoggerS3Logger.java │ │ │ ├── SettingsProcessor.java │ │ │ └── S3CompatibleArtifactsPublisher.java │ │ │ └── S3AdditionalHeadersProvider.java │ │ └── resources │ │ └── META-INF │ │ └── build-agent-plugin-s3-storage.xml ├── teamcity-plugin.xml └── build.gradle ├── .gitignore ├── settings.gradle ├── .editorconfig ├── lens-integration ├── build.gradle └── src │ ├── main │ └── java │ │ └── jetbrains │ │ └── buildServer │ │ └── artifacts │ │ └── s3 │ │ └── lens │ │ └── integration │ │ ├── LensIntegrationService.java │ │ ├── LensResponseErrorHandler.java │ │ ├── dto │ │ └── UploadInfoEvent.java │ │ └── JsonEntityProducer.java │ └── test │ └── java │ └── jetbrains │ └── buildServer │ └── artifacts │ └── s3 │ └── lens │ └── integration │ └── TestJsonGeneration.java ├── modules ├── s3-artifact-storage-common.iml ├── lens-integration.iml ├── s3-artifact-storage-agent.iml └── teamcity-s3-sdk.iml └── gradlew.bat /gradle.properties: -------------------------------------------------------------------------------- 1 | jacksonVersion=2.14.3 2 | -------------------------------------------------------------------------------- /s3-artifact-storage-common/src/test/resources/S3UtilsTest/file.txt: -------------------------------------------------------------------------------- 1 | My file -------------------------------------------------------------------------------- /s3-artifact-storage-ui/.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | !.eslintrc.js 3 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules 2 | yarn-error.log 3 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true 3 | } 4 | -------------------------------------------------------------------------------- /s3-artifact-storage-common/src/test/resources/S3UtilsTest/file.css: -------------------------------------------------------------------------------- 1 | 2 | 3 | body { 4 | padding: 0; 5 | } 6 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/.npmrc: -------------------------------------------------------------------------------- 1 | @jetbrains-internal:registry= https://packages.jetbrains.team/npm/p/tc/teamcity-npm 2 | -------------------------------------------------------------------------------- /s3-artifact-storage-server/src/main/resources/buildServerResources/.gitignore: -------------------------------------------------------------------------------- 1 | bundle-report.html 2 | bundle.js 3 | bundle.js.LICENSE.txt 4 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains/teamcity-s3-artifact-storage-plugin/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /s3-artifact-storage-common/src/test/resources/S3UtilsTest/file.htm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /s3-artifact-storage-common/src/test/resources/S3UtilsTest/file.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/S3Dto.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3; 2 | 3 | public interface S3Dto { 4 | } 5 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": ["@babel/preset-env", "@babel/preset-react", "@babel/preset-typescript"], 3 | "plugins": [ 4 | ["@babel/transform-runtime"] 5 | ] 6 | } -------------------------------------------------------------------------------- /s3-artifact-storage-agent/src/test/resources/artifacts/file.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains/teamcity-s3-artifact-storage-plugin/HEAD/s3-artifact-storage-agent/src/test/resources/artifacts/file.zip -------------------------------------------------------------------------------- /s3-artifact-storage-common/src/test/resources/S3UtilsTest/file.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains/teamcity-s3-artifact-storage-plugin/HEAD/s3-artifact-storage-common/src/test/resources/S3UtilsTest/file.bin -------------------------------------------------------------------------------- /s3-artifact-storage-common/src/test/resources/S3UtilsTest/file.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains/teamcity-s3-artifact-storage-plugin/HEAD/s3-artifact-storage-common/src/test/resources/S3UtilsTest/file.jpg -------------------------------------------------------------------------------- /s3-artifact-storage-common/src/test/resources/S3UtilsTest/file.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains/teamcity-s3-artifact-storage-plugin/HEAD/s3-artifact-storage-common/src/test/resources/S3UtilsTest/file.zip -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | .idea/ 3 | target/ 4 | s3-artifact-storage-agent/out/* 5 | s3-artifact-storage-common/out/* 6 | s3-artifact-storage-server/out/* 7 | .gradle/* 8 | build/ 9 | .DS_Store 10 | local-repo 11 | local-aws-repo 12 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 's3-artifact-storage' 2 | 3 | include 's3-artifact-storage-agent' 4 | include 's3-artifact-storage-common' 5 | include 's3-artifact-storage-server' 6 | include 'teamcity-s3-sdk' 7 | include 'lens-integration' 8 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # top-most EditorConfig file 2 | root = true 3 | 4 | # Unix-style newlines with a newline ending every file 5 | [*] 6 | end_of_line = lf 7 | insert_final_newline = true 8 | indent_style = space 9 | 10 | [*.java] 11 | indent_size = 2 -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /s3-artifact-storage-agent/src/main/java/jetbrains/buildServer/artifacts/s3/download/IORunnable.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.download; 2 | 3 | import java.io.IOException; 4 | 5 | public interface IORunnable { 6 | void run() throws IOException; 7 | } 8 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/types/declarations.d.ts: -------------------------------------------------------------------------------- 1 | declare module '@jetbrains/logos/*'; 2 | declare module '@jetbrains/icons/*'; 3 | 4 | declare module '*.css' { 5 | const content: { readonly [className: string]: string }; 6 | export default content; 7 | } 8 | 9 | declare module '*.svg'; 10 | -------------------------------------------------------------------------------- /s3-artifact-storage-server/src/main/java/jetbrains/buildServer/artifacts/s3/cleanup/CleanupListener.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.cleanup; 2 | 3 | public interface CleanupListener { 4 | void onError(Exception exception, boolean isRecoverable); 5 | void onSuccess(String objectKey); 6 | } 7 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/test/java/testng.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/index.tsx: -------------------------------------------------------------------------------- 1 | import { React, ReactDOM } from '@jetbrains/teamcity-api'; 2 | 3 | import App from './App/App'; 4 | import { Config } from './types'; 5 | 6 | global.renderEditS3Storage = (config: Config) => { 7 | ReactDOM.render( 8 | , 9 | document.getElementById('edit-s3-storage-root') 10 | ); 11 | }; 12 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/third-party-licenses-json.js: -------------------------------------------------------------------------------- 1 | module.exports = ({modules}) => 2 | JSON.stringify( 3 | modules.map(({license, ...rest}) => ({ 4 | ...rest, 5 | license: license.name, 6 | licenseUrl: license.url, 7 | })), 8 | null, 9 | // eslint-disable-next-line no-magic-numbers 10 | 2, 11 | ) 12 | -------------------------------------------------------------------------------- /s3-artifact-storage-server/src/test/java/testng-docker-testcontainers.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /s3-artifact-storage-server/src/main/java/jetbrains/buildServer/artifacts/s3/cleanup/AbstractCleanupListener.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.cleanup; 2 | 3 | public class AbstractCleanupListener implements CleanupListener{ 4 | @Override 5 | public void onError(Exception exception, boolean isRecoverable) { 6 | } 7 | 8 | @Override 9 | public void onSuccess(String objectKey) { 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./node_modules/@jetbrains/teamcity-api/plugin.tsconfig.json", 3 | "include": [ 4 | "./src/**/*" 5 | ], 6 | "compilerOptions": { 7 | "strict": true, 8 | "esModuleInterop": true, 9 | "lib": [ 10 | "dom", 11 | "es2015" 12 | ], 13 | "jsx": "react-jsx", 14 | "typeRoots": ["./node_modules/@types", "./src/types"] 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /s3-artifact-storage-agent/teamcity-plugin.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/publish/presigned/upload/PublishingInterruptedException.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish.presigned.upload; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class PublishingInterruptedException extends RuntimeException { 6 | public PublishingInterruptedException(@NotNull final String reason) { 7 | super(reason); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/amazonClient/WithS3Client.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.amazonClient; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | import org.jetbrains.annotations.Nullable; 5 | import software.amazon.awssdk.services.s3.S3Client; 6 | 7 | public interface WithS3Client { 8 | @Nullable 9 | T execute(@NotNull S3Client client) throws E; 10 | } 11 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/publish/logger/S3UploadLogger.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish.logger; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public interface S3UploadLogger { 6 | void debug(@NotNull String message); 7 | 8 | void info(@NotNull String message); 9 | 10 | void warn(@NotNull String message); 11 | 12 | void error(@NotNull String message); 13 | } 14 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/publish/errors/ResponseAdapter.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish.errors; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | import org.jetbrains.annotations.Nullable; 5 | 6 | public interface ResponseAdapter { 7 | 8 | @Nullable 9 | String getHeader(@NotNull final String header); 10 | 11 | @Nullable 12 | String getResponse(); 13 | 14 | int getStatusCode(); 15 | } 16 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/amazonClient/WithS3Presigner.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.amazonClient; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | import org.jetbrains.annotations.Nullable; 5 | import software.amazon.awssdk.services.s3.presigner.S3Presigner; 6 | 7 | public interface WithS3Presigner { 8 | @Nullable 9 | T execute(@NotNull S3Presigner presigner) throws E; 10 | } 11 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/cloudfront/CloudFrontUtils.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.cloudfront; 2 | 3 | public class CloudFrontUtils { 4 | 5 | public enum Protocol { 6 | http, 7 | https 8 | } 9 | 10 | public static String generateResourcePath(Protocol protocol, String distributionDomain, String resourcePath) { 11 | return protocol + "://" + distributionDomain + "/" + resourcePath; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /lens-integration/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java-library' 3 | } 4 | 5 | dependencies { 6 | implementation project(':teamcity-s3-sdk') 7 | 8 | compileOnly "org.jetbrains.teamcity:agent-openapi:${teamcityVersion}" 9 | compileOnly "org.jetbrains.teamcity:web-openapi:${teamcityVersion}" 10 | testImplementation "org.testng:testng:6.8.21" 11 | testImplementation "org.jetbrains.teamcity:tests-support:${teamcityVersion}" 12 | testImplementation group: 'org.mockito', name: 'mockito-core', version: '3.12.1' 13 | } 14 | -------------------------------------------------------------------------------- /s3-artifact-storage-agent/src/main/java/jetbrains/buildServer/artifacts/s3/download/parallel/splitter/FileSplitter.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.download.parallel.splitter; 2 | 3 | import java.util.List; 4 | import jetbrains.buildServer.artifacts.s3.download.parallel.FilePart; 5 | import org.jetbrains.annotations.NotNull; 6 | 7 | public interface FileSplitter { 8 | 9 | @NotNull 10 | List split(long fileSize); 11 | 12 | @NotNull 13 | SplitabilityReport testSplitability(long fileSize); 14 | } 15 | -------------------------------------------------------------------------------- /s3-artifact-storage-common/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java-library' 3 | } 4 | 5 | dependencies { 6 | compileOnly "org.jetbrains.teamcity:common-api:${teamcityVersion}" 7 | api project(':teamcity-s3-sdk') 8 | testImplementation "org.testng:testng:6.8.21" 9 | testImplementation "org.jetbrains.teamcity:tests-support:${teamcityVersion}" 10 | testImplementation group: 'org.mockito', name: 'mockito-core', version: '3.12.1' 11 | testImplementation("org.jetbrains.teamcity.plugins:aws-core-common:$awsCoreVersion") { 12 | changing = true 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/serialization/XmlSerializer.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.serialization; 2 | 3 | 4 | import com.fasterxml.jackson.core.JsonProcessingException; 5 | import org.jdom.Element; 6 | import org.jetbrains.annotations.NotNull; 7 | 8 | public interface XmlSerializer { 9 | 10 | String serialize(@NotNull Object object) throws JsonProcessingException; 11 | 12 | Element serializeAsElement(@NotNull Object object); 13 | 14 | T deserialize(@NotNull String xml, @NotNull Class clazz); 15 | } 16 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/PresignedUrlWithTtl.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class PresignedUrlWithTtl { 6 | private final String url; 7 | private final int urlTtlSeconds; 8 | 9 | public PresignedUrlWithTtl(@NotNull String url, int urlTtlSeconds) { 10 | this.url = url; 11 | this.urlTtlSeconds = urlTtlSeconds; 12 | } 13 | 14 | public String getUrl() { 15 | return url; 16 | } 17 | 18 | public int getUrlTtlSeconds() { 19 | return urlTtlSeconds; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/amazonClient/WithCloudFrontClient.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.amazonClient; 2 | 3 | import jetbrains.buildServer.serverSide.connections.credentials.ConnectionCredentialsException; 4 | import org.jetbrains.annotations.NotNull; 5 | import org.jetbrains.annotations.Nullable; 6 | import software.amazon.awssdk.services.cloudfront.CloudFrontClient; 7 | 8 | public interface WithCloudFrontClient { 9 | @Nullable 10 | T execute(@NotNull CloudFrontClient client) throws E, ConnectionCredentialsException; 11 | } 12 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/App/MultipartUpload/components/CustomizeUpload.tsx: -------------------------------------------------------------------------------- 1 | import { React } from '@jetbrains/teamcity-api'; 2 | import { FormCheckbox } from '@jetbrains-internal/tcci-react-ui-components'; 3 | import { useFormContext } from 'react-hook-form'; 4 | 5 | import { FormFields } from '../../appConstants'; 6 | 7 | export default function CustomizeUpload() { 8 | const { control } = useFormContext(); 9 | return ( 10 | 15 | ); 16 | } 17 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/types/global.d.ts: -------------------------------------------------------------------------------- 1 | export {}; 2 | 3 | declare global { 4 | function renderEditS3Storage(config: ConfigWrapper): void; 5 | 6 | interface Window { 7 | BS: { 8 | Encrypt: { 9 | encryptData: (value: string, publicKey: string) => string; 10 | }; 11 | Util: { 12 | showHelp: ( 13 | event: React.MouseEvent, 14 | href: string, 15 | { width: number, height: number } 16 | ) => void; 17 | }; 18 | }; 19 | $j: JQueryStatic; 20 | __secretKey: string | null | undefined; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/App/S3/components/IAMRole.tsx: -------------------------------------------------------------------------------- 1 | import { React } from '@jetbrains/teamcity-api'; 2 | import { 3 | FormInput, 4 | FormRow, 5 | } from '@jetbrains-internal/tcci-react-ui-components'; 6 | 7 | import { useFormContext } from 'react-hook-form'; 8 | 9 | import { IFormInput } from '../../../types'; 10 | import { FormFields } from '../../appConstants'; 11 | 12 | export default function IAMRole() { 13 | const { control } = useFormContext(); 14 | return ( 15 | 16 | 17 | 18 | ); 19 | } 20 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/App/components/BucketPrefix.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | FormInput, 3 | FormRow, 4 | } from '@jetbrains-internal/tcci-react-ui-components'; 5 | import { React } from '@jetbrains/teamcity-api'; 6 | import { useFormContext } from 'react-hook-form'; 7 | 8 | import { FormFields } from '../appConstants'; 9 | 10 | export default function BucketPrefix() { 11 | const { control } = useFormContext(); 12 | return ( 13 | 18 | 19 | 20 | ); 21 | } 22 | -------------------------------------------------------------------------------- /s3-artifact-storage-common/src/test/java/testng.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/publish/presigned/upload/PresignedUrlsProviderClientFactory.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish.presigned.upload; 2 | 3 | import java.util.Collection; 4 | import jetbrains.buildServer.artifacts.ArtifactTransportAdditionalHeadersProvider; 5 | import org.jetbrains.annotations.NotNull; 6 | 7 | public interface PresignedUrlsProviderClientFactory { 8 | PresignedUrlsProviderClient createClient(@NotNull final TeamCityConnectionConfiguration connectionConfiguration, 9 | @NotNull Collection additionalHeadersProviders); 10 | } 11 | -------------------------------------------------------------------------------- /s3-artifact-storage-server/src/main/resources/buildServerResources/s3_storage_settings.jsp: -------------------------------------------------------------------------------- 1 | 2 | <%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core" %> 3 | <%@ taglib prefix="intprop" uri="/WEB-INF/functions/intprop" %> 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/hooks/useStorageOptions.tsx: -------------------------------------------------------------------------------- 1 | import { Option } from '@jetbrains-internal/tcci-react-ui-components'; 2 | 3 | import { useAppContext } from '../contexts/AppContext'; 4 | 5 | export default function useStorageOptions() { 6 | const config = useAppContext(); 7 | 8 | const storageTypes = config.storageTypes 9 | .split(/[\[\],]/) 10 | .map((it) => it.trim()) 11 | .filter((it) => !!it); 12 | const storageNames = config.storageNames 13 | .split(/[\[\],]/) 14 | .map((it) => it.trim()) 15 | .filter((it) => !!it); 16 | return storageTypes.reduce((acc, next, i) => { 17 | acc.push({ key: next, label: storageNames[i] }); 18 | return acc; 19 | }, []); 20 | } 21 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/App/S3/components/DefaultProviderChain.tsx: -------------------------------------------------------------------------------- 1 | import { React } from '@jetbrains/teamcity-api'; 2 | import { 3 | FormCheckbox, 4 | FormRow, 5 | } from '@jetbrains-internal/tcci-react-ui-components'; 6 | 7 | import { useFormContext } from 'react-hook-form'; 8 | 9 | import { FormFields } from '../../appConstants'; 10 | import { IFormInput } from '../../../types'; 11 | 12 | export default function DefaultProviderChain() { 13 | const { control } = useFormContext(); 14 | 15 | return ( 16 | 17 | 21 | 22 | ); 23 | } 24 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/App/Storage/StorageSection.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | Option, 3 | SectionHeader, 4 | } from '@jetbrains-internal/tcci-react-ui-components'; 5 | 6 | import { React } from '@jetbrains/teamcity-api'; 7 | 8 | import StorageType from './components/StorageType'; 9 | import StorageName from './components/StorageName'; 10 | import StorageId from './components/StorageId'; 11 | 12 | export default function StorageSection({ 13 | onReset, 14 | }: { 15 | onReset: (option: Option | null) => void | undefined; 16 | }) { 17 | return ( 18 |
19 | {'Storage'} 20 | 21 | 22 | 23 |
24 | ); 25 | } 26 | -------------------------------------------------------------------------------- /s3-artifact-storage-agent/src/main/java/jetbrains/buildServer/artifacts/s3/download/parallel/PartFailure.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.download.parallel; 2 | 3 | 4 | import java.io.IOException; 5 | import org.jetbrains.annotations.NotNull; 6 | 7 | public class PartFailure { 8 | @NotNull 9 | private final FilePart myPart; 10 | @NotNull 11 | private final IOException myException; 12 | 13 | public PartFailure(@NotNull FilePart part, @NotNull IOException exception) { 14 | myPart = part; 15 | myException = exception; 16 | } 17 | 18 | @NotNull 19 | public FilePart getPart() { 20 | return myPart; 21 | } 22 | 23 | @NotNull 24 | public IOException getException() { 25 | return myException; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/Utilities/fetchHelper.ts: -------------------------------------------------------------------------------- 1 | import { utils } from '@jetbrains/teamcity-api'; 2 | 3 | const request = async ( 4 | url: string, 5 | params: { [k: string]: string } | null, 6 | method = 'GET' 7 | ) => { 8 | let body = null; 9 | if (params) { 10 | body = new FormData(); 11 | 12 | for (const key in params) { 13 | const value = params[key]; 14 | body.append(key, value); 15 | } 16 | } 17 | 18 | return await utils.requestText( 19 | url.replace(/^\/+/, ''), 20 | { 21 | method, 22 | body, 23 | }, 24 | true 25 | ); 26 | }; 27 | 28 | export const post = async ( 29 | url: string, 30 | params: { [k: string]: string } | null = null 31 | ) => await request(url, params, 'POST'); 32 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/App/S3Compatible/S3Section.tsx: -------------------------------------------------------------------------------- 1 | import { React } from '@jetbrains/teamcity-api'; 2 | import { SectionHeader } from '@jetbrains-internal/tcci-react-ui-components'; 3 | 4 | import Bucket from '../components/Bucket'; 5 | import BucketPrefix from '../components/BucketPrefix'; 6 | 7 | import AccessKeyId from './components/AccessKeyId'; 8 | import Endpoint from './components/Endpoint'; 9 | import SecretAccessKey from './components/SecretAccessKey'; 10 | 11 | export default function S3Section() { 12 | return ( 13 |
14 | {'S3'} 15 | 16 | 17 | 18 | 19 | 20 |
21 | ); 22 | } 23 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/App/S3Compatible/components/AccessKeyId.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | FormInput, 3 | FormRow, 4 | } from '@jetbrains-internal/tcci-react-ui-components'; 5 | import { React } from '@jetbrains/teamcity-api'; 6 | import { useFormContext } from 'react-hook-form'; 7 | 8 | import { FormFields } from '../../appConstants'; 9 | 10 | export default function AccessKeyId() { 11 | const { control } = useFormContext(); 12 | 13 | return ( 14 | 15 | 22 | 23 | ); 24 | } 25 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/publish/S3FileUploaderFactory.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish; 2 | 3 | import java.util.function.Supplier; 4 | import jetbrains.buildServer.artifacts.s3.S3Configuration; 5 | import jetbrains.buildServer.artifacts.s3.publish.logger.S3UploadLogger; 6 | import jetbrains.buildServer.artifacts.s3.publish.presigned.upload.PresignedUrlsProviderClient; 7 | import org.jetbrains.annotations.NotNull; 8 | 9 | public interface S3FileUploaderFactory { 10 | S3FileUploader create(@NotNull S3Configuration s3Configuration, 11 | @NotNull S3UploadLogger s3UploadLogger, 12 | @NotNull Supplier presignedUrlsProviderClientSupplier); 13 | } 14 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/serialization/impl/JDomElementDeserializer.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.serialization.impl; 2 | 3 | import java.io.IOException; 4 | import java.io.StringReader; 5 | import org.jdom.Element; 6 | import org.jdom.JDOMException; 7 | import org.jdom.input.SAXBuilder; 8 | import org.jetbrains.annotations.NotNull; 9 | 10 | public class JDomElementDeserializer { 11 | @NotNull 12 | public Element deserialize(@NotNull final String xml) { 13 | final StringReader stringReader = new StringReader(xml); 14 | try { 15 | return new SAXBuilder().build(stringReader).detachRootElement(); 16 | } catch (JDOMException | IOException e) { 17 | throw new RuntimeException(e); 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /s3-artifact-storage-server/src/main/java/jetbrains/buildServer/artifacts/s3/S3CompatibleArtifactContentProvider.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3; 2 | 3 | import jetbrains.buildServer.artifacts.s3.amazonClient.AmazonS3Provider; 4 | import jetbrains.buildServer.serverSide.ServerPaths; 5 | import org.jetbrains.annotations.NotNull; 6 | 7 | public class S3CompatibleArtifactContentProvider extends S3ArtifactContentProvider { 8 | public S3CompatibleArtifactContentProvider(@NotNull ServerPaths serverPaths, 9 | @NotNull AmazonS3Provider amazonS3Provider) { 10 | super(serverPaths, amazonS3Provider); 11 | } 12 | 13 | @NotNull 14 | @Override 15 | public String getType() { 16 | return S3Constants.S3_COMPATIBLE_STORAGE_TYPE; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /s3-artifact-storage-server/src/main/java/jetbrains/buildServer/artifacts/s3/orphans/OrphanedArtifacts.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.orphans; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.Collection; 6 | import java.util.List; 7 | 8 | public class OrphanedArtifacts { 9 | private final Collection orphanedPaths; 10 | private final List errors; 11 | 12 | OrphanedArtifacts(@NotNull Collection orphanedPaths, @NotNull List errors) { 13 | this.orphanedPaths = orphanedPaths; 14 | this.errors = errors; 15 | } 16 | 17 | public Collection getOrphanedPaths() { 18 | return orphanedPaths; 19 | } 20 | 21 | public List getErrors() { 22 | return errors; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /s3-artifact-storage-server/teamcity-plugin.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | s3-artifact-storage 7 | Amazon S3 Artifact Storage 8 | @Version@ 9 | Allows storing build artifacts in an Amazon S3 bucket 10 | 11 | JetBrains 12 | http://www.jetbrains.com 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /s3-artifact-storage-agent/src/test/java/testng.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /s3-artifact-storage-agent/src/main/java/jetbrains/buildServer/artifacts/s3/download/parallel/strategy/ParallelDownloadStrategy.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.download.parallel.strategy; 2 | 3 | import java.io.IOException; 4 | import java.nio.file.Path; 5 | import jetbrains.buildServer.artifacts.FileProgress; 6 | import jetbrains.buildServer.artifacts.s3.download.parallel.ParallelDownloadContext; 7 | import org.jetbrains.annotations.NotNull; 8 | 9 | public interface ParallelDownloadStrategy { 10 | void download(@NotNull String srcUrl, 11 | @NotNull Path targetFile, 12 | long fileSize, 13 | @NotNull FileProgress downloadProgress, 14 | @NotNull ParallelDownloadContext downloadContext) throws IOException; 15 | 16 | @NotNull 17 | String getName(); 18 | } 19 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/App/MultipartUpload/components/PartSize.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | FormInput, 3 | FormRow, 4 | } from '@jetbrains-internal/tcci-react-ui-components'; 5 | import { React } from '@jetbrains/teamcity-api'; 6 | import { useFormContext } from 'react-hook-form'; 7 | 8 | import { FormFields } from '../../appConstants'; 9 | 10 | export default function PartSize() { 11 | const { control } = useFormContext(); 12 | return ( 13 | 17 | 22 | 23 | ); 24 | } 25 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/cloudfront/RequestMetadata.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.cloudfront; 2 | 3 | import org.jetbrains.annotations.Nullable; 4 | 5 | public class RequestMetadata { 6 | private final String region; 7 | private final String userAgent; 8 | 9 | public RequestMetadata(@Nullable String region, @Nullable String userAgent) { 10 | this.region = region; 11 | this.userAgent = userAgent; 12 | } 13 | 14 | @Nullable 15 | public String getRegion() { 16 | return region; 17 | } 18 | 19 | @Nullable 20 | public String getUserAgent() { 21 | return userAgent; 22 | } 23 | 24 | public static RequestMetadata from(@Nullable String region, @Nullable String userAgent) { 25 | return new RequestMetadata(region, userAgent); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/serialization/S3XmlSerializerFactory.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.serialization; 2 | 3 | import jetbrains.buildServer.artifacts.s3.serialization.impl.XmlSerializerImpl; 4 | import jetbrains.buildServer.util.impl.Lazy; 5 | import org.jetbrains.annotations.NotNull; 6 | 7 | public final class S3XmlSerializerFactory { 8 | @NotNull 9 | private static final Lazy OUR_SERIALIZER = new Lazy() { 10 | @NotNull 11 | @Override 12 | protected XmlSerializerImpl createValue() { 13 | return new XmlSerializerImpl(); 14 | } 15 | }; 16 | 17 | private S3XmlSerializerFactory() { 18 | } 19 | 20 | @NotNull 21 | public static XmlSerializer getInstance() { 22 | return OUR_SERIALIZER.get(); 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/App/S3Compatible/components/SecretAccessKey.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | FormInput, 3 | FormRow, 4 | } from '@jetbrains-internal/tcci-react-ui-components'; 5 | import { React } from '@jetbrains/teamcity-api'; 6 | import { useFormContext } from 'react-hook-form'; 7 | 8 | import { FormFields } from '../../appConstants'; 9 | 10 | export default function SecretAccessKey() { 11 | const { control } = useFormContext(); 12 | 13 | return ( 14 | 19 | 27 | 28 | ); 29 | } 30 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/cloudfront/CloudFrontPresignedUrlProvider.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.cloudfront; 2 | 3 | import java.io.IOException; 4 | import jetbrains.buildServer.artifacts.s3.PresignedUrlWithTtl; 5 | import org.jetbrains.annotations.NotNull; 6 | import org.jetbrains.annotations.Nullable; 7 | 8 | public interface CloudFrontPresignedUrlProvider { 9 | 10 | @Nullable 11 | PresignedUrlWithTtl generateDownloadUrl(@NotNull String objectKey, @NotNull CloudFrontSettings settings) throws IOException; 12 | 13 | String generateUploadUrl(@NotNull String objectKey, @NotNull CloudFrontSettings settings) throws IOException; 14 | 15 | String generateUploadUrlForPart(@NotNull String objectKey, int nPart, @NotNull String uploadId, @NotNull CloudFrontSettings settings) throws IOException; 16 | } 17 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/publish/errors/HttpResponseErrorHandler.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish.errors; 2 | 3 | import java.util.Arrays; 4 | import java.util.HashSet; 5 | import jetbrains.buildServer.artifacts.s3.publish.presigned.util.HttpClientUtil; 6 | import org.jetbrains.annotations.NotNull; 7 | 8 | public interface HttpResponseErrorHandler { 9 | @NotNull 10 | static final HashSet OUR_RECOVERABLE_STATUS_CODES = new HashSet<>(Arrays.asList(0, 500, 502, 503, 504)); 11 | 12 | public boolean canHandle(@NotNull ResponseAdapter responseWrapper); 13 | 14 | @NotNull 15 | public HttpClientUtil.HttpErrorCodeException handle(@NotNull ResponseAdapter responseWrapper); 16 | 17 | @NotNull 18 | public default String name() { 19 | return getClass().getName(); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/S3Settings.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3; 2 | 3 | import java.util.Map; 4 | import org.jetbrains.annotations.NotNull; 5 | import org.jetbrains.annotations.Nullable; 6 | import software.amazon.awssdk.services.s3.model.ObjectCannedACL; 7 | 8 | import static jetbrains.buildServer.artifacts.s3.S3Constants.PROJECT_ID_PARAM; 9 | 10 | public interface S3Settings { 11 | @NotNull 12 | String getBucketName(); 13 | 14 | int getUrlTtlSeconds(); 15 | 16 | int getUrlExtendedTtlSeconds(); 17 | 18 | @NotNull 19 | ObjectCannedACL getAcl(); 20 | 21 | Map toRawSettings(); 22 | 23 | @NotNull 24 | Map getProjectSettings(); 25 | 26 | void setTtl(long ttl); 27 | 28 | @Nullable 29 | default String getProjectId() { return getProjectSettings().get(PROJECT_ID_PARAM); } 30 | } 31 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/publish/logger/S3Log4jUploadLogger.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish.logger; 2 | 3 | 4 | import com.intellij.openapi.diagnostic.Logger; 5 | import org.jetbrains.annotations.NotNull; 6 | 7 | public class S3Log4jUploadLogger implements S3UploadLogger { 8 | @NotNull 9 | private static final Logger LOGGER = Logger.getInstance(S3Log4jUploadLogger.class); 10 | 11 | @Override 12 | public void debug(@NotNull String message) { 13 | LOGGER.debug(message); 14 | } 15 | 16 | @Override 17 | public void info(@NotNull String message) { 18 | LOGGER.info(message); 19 | } 20 | 21 | @Override 22 | public void warn(@NotNull String message) { 23 | LOGGER.warn(message); 24 | } 25 | 26 | @Override 27 | public void error(@NotNull String message) { 28 | LOGGER.error(message); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/exceptions/InvalidSettingsException.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.exceptions; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | import jetbrains.buildServer.util.StringUtil; 6 | import org.jetbrains.annotations.NotNull; 7 | 8 | public class InvalidSettingsException extends RuntimeException { 9 | 10 | private static final long serialVersionUID = 1L; 11 | 12 | @NotNull 13 | private final Map myInvalids; 14 | 15 | public InvalidSettingsException(@NotNull Map invalids) { 16 | myInvalids = new HashMap<>(invalids); 17 | } 18 | 19 | @Override 20 | public String getMessage() { 21 | return StringUtil.join("\n", myInvalids.values()); 22 | } 23 | 24 | @NotNull 25 | public Map getInvalids() { 26 | return myInvalids; 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /lens-integration/src/main/java/jetbrains/buildServer/artifacts/s3/lens/integration/LensIntegrationService.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.lens.integration; 2 | 3 | import java.time.Duration; 4 | import java.util.Collection; 5 | import jetbrains.buildServer.agent.AgentRunningBuild; 6 | import jetbrains.buildServer.artifacts.s3.publish.UploadStatistics; 7 | import jetbrains.buildServer.artifacts.s3.publish.presigned.upload.TeamCityConnectionConfiguration; 8 | import org.jetbrains.annotations.NotNull; 9 | 10 | public interface LensIntegrationService { 11 | void generateUploadEvents(@NotNull final AgentRunningBuild build, 12 | @NotNull final Collection statistics, 13 | @NotNull final Duration totalUploadDuration, 14 | @NotNull final TeamCityConnectionConfiguration teamCityConnectionConfiguration); 15 | } 16 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/publish/presigned/upload/TeamCityServerPresignedUrlsProviderClientFactory.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish.presigned.upload; 2 | 3 | import java.util.Collection; 4 | import jetbrains.buildServer.artifacts.ArtifactTransportAdditionalHeadersProvider; 5 | import org.jetbrains.annotations.NotNull; 6 | 7 | public class TeamCityServerPresignedUrlsProviderClientFactory implements PresignedUrlsProviderClientFactory { 8 | @Override 9 | public PresignedUrlsProviderClient createClient(@NotNull TeamCityConnectionConfiguration teamCityConnectionConfiguration, 10 | @NotNull Collection additionalHeadersProviders) { 11 | return new TeamCityServerPresignedUrlsProviderClient(teamCityConnectionConfiguration, additionalHeadersProviders); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/cloudfront/CloudFrontSettings.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.cloudfront; 2 | 3 | import jetbrains.buildServer.artifacts.s3.S3Settings; 4 | import org.jetbrains.annotations.NotNull; 5 | import org.jetbrains.annotations.Nullable; 6 | 7 | public interface CloudFrontSettings extends S3Settings { 8 | boolean getCloudFrontEnabled(); 9 | 10 | @Deprecated 11 | @NotNull 12 | String getCloudFrontDistribution(); 13 | 14 | @Nullable 15 | String getCloudFrontUploadDistribution(); 16 | 17 | @Nullable 18 | String getCloudFrontDownloadDistribution(); 19 | 20 | @NotNull 21 | String getCloudFrontPublicKeyId(); 22 | 23 | @NotNull 24 | String getCloudFrontPrivateKey(); 25 | 26 | @NotNull 27 | String getBucketRegion(); 28 | 29 | @Nullable 30 | String getRequestRegion(); 31 | 32 | @Nullable 33 | String getRequestUserAgent(); 34 | } 35 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const getWebpackConfig = require('@jetbrains/teamcity-api/getWebpackConfig'); 3 | const LicenseChecker = require('@jetbrains/ring-ui-license-checker'); 4 | 5 | function createLicenseChecker(filename) { 6 | return new LicenseChecker({ 7 | format: require('./third-party-licenses-json'), 8 | filename, 9 | exclude: [/@jetbrains/], 10 | surviveLicenseErrors: true, 11 | }); 12 | } 13 | 14 | const createConfig = getWebpackConfig({ 15 | srcPath: path.join(__dirname, './src'), 16 | outputPath: path.resolve(__dirname, '../s3-artifact-storage-server/src/main/resources/buildServerResources'), 17 | entry: './src/index.tsx', 18 | useTypeScript: true 19 | }); 20 | 21 | const config = createConfig(); 22 | config.plugins.push(createLicenseChecker('../../../../../s3-artifact-storage-ui/js-related-libraries.json')) 23 | 24 | module.exports = config; 25 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/publish/presigned/upload/PresignedUploadProgressListener.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish.presigned.upload; 2 | 3 | import java.time.Duration; 4 | import org.jetbrains.annotations.NotNull; 5 | 6 | public interface PresignedUploadProgressListener { 7 | void onPartUploadFailed(@NotNull final Throwable e, int partIndex); 8 | 9 | void onPartUploadSuccess(@NotNull String uploadUrl, int partIndex, String digest); 10 | 11 | void onFileUploadFailed(@NotNull String message, boolean isRecoverable); 12 | 13 | void onFileUploadSuccess(String digest); 14 | 15 | void beforeUploadStarted(); 16 | 17 | void beforePartUploadStarted(int partIndex, long partSize); 18 | 19 | void setUpload(@NotNull S3PresignedUpload upload); 20 | 21 | void partsSeparated(int nParts, long chunkSizeInBytes, @NotNull Duration ofMillis); 22 | 23 | void urlsGenerated(@NotNull Duration ofMillis); 24 | } 25 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/App/MultipartUpload/MultipartUploadSection.tsx: -------------------------------------------------------------------------------- 1 | import { React } from '@jetbrains/teamcity-api'; 2 | import { SectionHeader } from '@jetbrains-internal/tcci-react-ui-components'; 3 | 4 | import { useFormContext } from 'react-hook-form'; 5 | 6 | import { FormFields } from '../appConstants'; 7 | 8 | import Threshold from './components/Threshold'; 9 | import PartSize from './components/PartSize'; 10 | import CustomizeUpload from './components/CustomizeUpload'; 11 | 12 | export default function MultipartUploadSection() { 13 | const { watch } = useFormContext(); 14 | const customizeUpload = watch(FormFields.CONNECTION_MULTIPART_CUSTOMIZE_FLAG); 15 | 16 | return ( 17 |
18 | {'Multipart upload settings'} 19 | 20 | {customizeUpload && ( 21 | <> 22 | 23 | 24 | 25 | )} 26 |
27 | ); 28 | } 29 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/hooks/useCanLoadBucketInfoData.tsx: -------------------------------------------------------------------------------- 1 | import { useFormContext } from 'react-hook-form'; 2 | 3 | import { FormFields } from '../App/appConstants'; 4 | import { AWS_S3, S3_COMPATIBLE } from '../App/Storage/components/StorageType'; 5 | import { IFormInput } from '../types'; 6 | 7 | export default function useCanLoadBucketInfoData() { 8 | const { watch } = useFormContext(); 9 | const awsConnectionId = watch(FormFields.AWS_CONNECTION_ID); 10 | const accessKeyId = watch(FormFields.ACCESS_KEY_ID); 11 | const secretAccessKey = watch(FormFields.SECRET_ACCESS_KEY); 12 | const currentType = watch(FormFields.STORAGE_TYPE); 13 | const isS3Compatible = currentType?.key === S3_COMPATIBLE; 14 | const isAwsS3 = currentType?.key === AWS_S3; 15 | const isDisabled = 16 | (isS3Compatible && (!accessKeyId || !secretAccessKey)) || 17 | (isAwsS3 && !awsConnectionId?.key && (!accessKeyId || !secretAccessKey)); 18 | 19 | return !isDisabled; 20 | } 21 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/App/S3/TransferSpeedUp/components/UploadDistribution.tsx: -------------------------------------------------------------------------------- 1 | import { React } from '@jetbrains/teamcity-api'; 2 | 3 | import { LabelWithHelp } from '@jetbrains-internal/tcci-react-ui-components'; 4 | 5 | import { FormFields } from '../../../appConstants'; 6 | import { useCloudFrontDistributionsContext } from '../contexts/CloudFrontDistributionsContext'; 7 | 8 | import Distribution from './Distribution'; 9 | 10 | function UploadDistributionLabel() { 11 | return ( 12 | 16 | ); 17 | } 18 | export default function UploadDistribution() { 19 | const { setUploadDistribution } = useCloudFrontDistributionsContext(); 20 | return ( 21 | } 23 | fieldName={FormFields.CLOUD_FRONT_UPLOAD_DISTRIBUTION} 24 | onChange={setUploadDistribution} 25 | /> 26 | ); 27 | } 28 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/App/S3/TransferSpeedUp/components/DownloadDistribution.tsx: -------------------------------------------------------------------------------- 1 | import { React } from '@jetbrains/teamcity-api'; 2 | 3 | import { LabelWithHelp } from '@jetbrains-internal/tcci-react-ui-components'; 4 | 5 | import { FormFields } from '../../../appConstants'; 6 | 7 | import { useCloudFrontDistributionsContext } from '../contexts/CloudFrontDistributionsContext'; 8 | 9 | import Distribution from './Distribution'; 10 | 11 | function DownloadDistributionLabel() { 12 | return ( 13 | 17 | ); 18 | } 19 | export default function DownloadDistribution() { 20 | const { setDownloadDistribution } = useCloudFrontDistributionsContext(); 21 | return ( 22 | } 24 | fieldName={FormFields.CLOUD_FRONT_DOWNLOAD_DISTRIBUTION} 25 | onChange={setDownloadDistribution} 26 | /> 27 | ); 28 | } 29 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/transport/MultipartUploadStartRequestDto.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.transport; 2 | 3 | import javax.xml.bind.annotation.XmlAccessType; 4 | import javax.xml.bind.annotation.XmlAccessorType; 5 | import javax.xml.bind.annotation.XmlRootElement; 6 | import jetbrains.buildServer.Used; 7 | import org.jetbrains.annotations.NotNull; 8 | 9 | @XmlRootElement(name = "multipartUploadStartRequest") 10 | @XmlAccessorType(XmlAccessType.PROPERTY) 11 | public class MultipartUploadStartRequestDto { 12 | private String objectKey; 13 | 14 | @Used("serialization") 15 | public MultipartUploadStartRequestDto() { 16 | } 17 | 18 | @Used("clients") 19 | public MultipartUploadStartRequestDto(@NotNull final String objectKey) { 20 | this.objectKey = objectKey; 21 | } 22 | 23 | public String getObjectKey() { 24 | return objectKey; 25 | } 26 | 27 | public void setObjectKey(String objectKey) { 28 | this.objectKey = objectKey; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /s3-artifact-storage-agent/src/main/java/jetbrains/buildServer/artifacts/s3/download/parallel/FilePart.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.download.parallel; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public final class FilePart { 6 | private final int myPartNumber; 7 | private final long myStartByte; 8 | private final long myEndByte; 9 | 10 | public FilePart(int partNumber, long startByte, long endByte) { 11 | myPartNumber = partNumber; 12 | myStartByte = startByte; 13 | myEndByte = endByte; 14 | } 15 | 16 | public int getPartNumber() { 17 | return myPartNumber; 18 | } 19 | 20 | public long getStartByte() { 21 | return myStartByte; 22 | } 23 | 24 | public long getEndByte() { 25 | return myEndByte; 26 | } 27 | 28 | public long getSizeBytes() { 29 | return myEndByte - myStartByte + 1; 30 | } 31 | 32 | @NotNull 33 | public String getDescription() { 34 | return String.format("%s (bytes %s-%s)", myPartNumber, myStartByte, myEndByte); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /s3-artifact-storage-agent/src/main/java/jetbrains/buildServer/artifacts/s3/publish/logger/BuildLoggerS3Logger.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish.logger; 2 | 3 | import jetbrains.buildServer.agent.FlowLogger; 4 | import org.jetbrains.annotations.NotNull; 5 | 6 | public class BuildLoggerS3Logger implements S3UploadLogger { 7 | private final FlowLogger myBuildProgressLogger; 8 | 9 | public BuildLoggerS3Logger(FlowLogger progressLogger) { 10 | myBuildProgressLogger = progressLogger; 11 | } 12 | 13 | @Override 14 | public void debug(@NotNull String message) { 15 | myBuildProgressLogger.debug(message); 16 | } 17 | 18 | @Override 19 | public void warn(@NotNull String message) { 20 | myBuildProgressLogger.warning(message); 21 | } 22 | 23 | @Override 24 | public void info(@NotNull String message) { 25 | myBuildProgressLogger.message(message); 26 | } 27 | 28 | @Override 29 | public void error(@NotNull String message) { 30 | myBuildProgressLogger.error(message); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /lens-integration/src/main/java/jetbrains/buildServer/artifacts/s3/lens/integration/LensResponseErrorHandler.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.lens.integration; 2 | 3 | import jetbrains.buildServer.artifacts.s3.publish.errors.HttpResponseErrorHandler; 4 | import jetbrains.buildServer.artifacts.s3.publish.errors.ResponseAdapter; 5 | import jetbrains.buildServer.artifacts.s3.publish.presigned.util.HttpClientUtil; 6 | import org.jetbrains.annotations.NotNull; 7 | 8 | class LensResponseErrorHandler implements HttpResponseErrorHandler { 9 | @Override 10 | public boolean canHandle(@NotNull ResponseAdapter responseWrapper) { 11 | // handle every error 12 | return true; 13 | } 14 | 15 | @NotNull 16 | @Override 17 | public HttpClientUtil.HttpErrorCodeException handle(@NotNull ResponseAdapter responseWrapper) { 18 | // just wrap it into HttpClientUtil.HttpErrorCodeException and return 19 | return new HttpClientUtil.HttpErrorCodeException(responseWrapper.getStatusCode(), responseWrapper.getResponse(), true); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /s3-artifact-storage-common/src/test/java/jetbrains/buildServer/artifacts/s3/S3UtilTest.java: -------------------------------------------------------------------------------- 1 | 2 | 3 | package jetbrains.buildServer.artifacts.s3; 4 | 5 | import java.io.File; 6 | import org.testng.Assert; 7 | import org.testng.annotations.DataProvider; 8 | import org.testng.annotations.Test; 9 | 10 | @Test 11 | public class S3UtilTest { 12 | @DataProvider 13 | public Object[][] getContentTypeData() { 14 | return new Object[][]{ 15 | {"file.zip", "application/zip"}, 16 | {"file.txt", "text/plain"}, 17 | {"file.jpg", "image/jpeg"}, 18 | {"file.png", "image/png"}, 19 | {"file.bin", "application/octet-stream"}, 20 | {"file.htm", "text/html"}, 21 | {"file.html", "text/html"}, 22 | {"file.css", "text/css"}, 23 | {"file.js", "application/javascript"}, 24 | }; 25 | } 26 | 27 | @Test(dataProvider = "getContentTypeData") 28 | public void getContentTypeTest(String fileName, String expectedType) { 29 | Assert.assertEquals(S3Util.getContentType(new File("S3UtilsTest", fileName)), expectedType); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/App/S3Compatible/components/Endpoint.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | FormInput, 3 | FormRow, 4 | } from '@jetbrains-internal/tcci-react-ui-components'; 5 | import { React } from '@jetbrains/teamcity-api'; 6 | import { useFormContext } from 'react-hook-form'; 7 | 8 | import { FormFields } from '../../appConstants'; 9 | 10 | export default function Endpoint() { 11 | const { control } = useFormContext(); 12 | const validate = (value: string) => { 13 | try { 14 | new URL(value); 15 | } catch (e) { 16 | return 'Invalid URL'; 17 | } 18 | return true; 19 | }; 20 | return ( 21 | 26 | 35 | 36 | ); 37 | } 38 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/FileUploadInfo.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | import org.jetbrains.annotations.Nullable; 5 | 6 | public class FileUploadInfo { 7 | @NotNull 8 | private final String myAbsolutePath; 9 | private final long mySize; 10 | @Nullable 11 | private final String myDigest; 12 | @NotNull 13 | private final String myArtifactPath; 14 | 15 | public FileUploadInfo(@NotNull final String artifactPath, @NotNull String absolutePath, final long size, @Nullable String digest) { 16 | myArtifactPath = artifactPath; 17 | myAbsolutePath = absolutePath; 18 | mySize = size; 19 | myDigest = digest; 20 | } 21 | 22 | public long getSize() { 23 | return mySize; 24 | } 25 | 26 | @NotNull 27 | public String getArtifactPath() { 28 | return myArtifactPath; 29 | } 30 | 31 | @NotNull 32 | public String getAbsolutePath() { 33 | return myAbsolutePath; 34 | } 35 | 36 | @Nullable 37 | public String getDigest() { 38 | return myDigest; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /s3-artifact-storage-agent/src/main/java/jetbrains/buildServer/artifacts/s3/download/parallel/splitter/SplitabilityReport.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.download.parallel.splitter; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | import org.jetbrains.annotations.Nullable; 5 | 6 | public final class SplitabilityReport { 7 | private final boolean myIsSplittable; 8 | @Nullable 9 | private final String myUnsplitablilityReason; 10 | 11 | private SplitabilityReport(boolean isSplittable, @Nullable String unsplitablilityReason) { 12 | myIsSplittable = isSplittable; 13 | myUnsplitablilityReason = unsplitablilityReason; 14 | } 15 | 16 | public boolean isSplittable() { 17 | return myIsSplittable; 18 | } 19 | 20 | @Nullable 21 | public String getUnsplitablilityReason() { 22 | return myUnsplitablilityReason; 23 | } 24 | 25 | public static SplitabilityReport splittable() { 26 | return new SplitabilityReport(true, null); 27 | } 28 | 29 | public static SplitabilityReport unsplittable(@NotNull String reason) { 30 | return new SplitabilityReport(false, reason); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/publish/presigned/util/Throwables.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish.presigned.util; 2 | 3 | // utility class 4 | // TODO: merge it with jetbrains.buildServer.util.ExceptionUtil 5 | public class Throwables { 6 | private Throwables() { 7 | } 8 | 9 | public static Throwable getRootCause(Throwable throwable) { 10 | // Keep a second pointer that slowly walks the causal chain. If the fast pointer ever catches 11 | // the slower pointer, then there's a loop. 12 | Throwable slowPointer = throwable; 13 | boolean advanceSlowPointer = false; 14 | 15 | Throwable cause; 16 | while ((cause = throwable.getCause()) != null) { 17 | throwable = cause; 18 | 19 | if (throwable == slowPointer) { 20 | throw new IllegalArgumentException("Loop in causal chain detected.", throwable); 21 | } 22 | if (advanceSlowPointer) { 23 | slowPointer = slowPointer.getCause(); 24 | } 25 | advanceSlowPointer = !advanceSlowPointer; // only advance every other iteration 26 | } 27 | return throwable; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/SSLParamUtil.java: -------------------------------------------------------------------------------- 1 | 2 | 3 | package jetbrains.buildServer.artifacts.s3; 4 | 5 | import jetbrains.buildServer.util.amazon.AWSCommonParams; 6 | import org.jetbrains.annotations.NotNull; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | /** 12 | * Utils class to add ssl parameter into params. 13 | * 14 | * @author Mikhail Khorkov 15 | * @since 2018.1 16 | */ 17 | public class SSLParamUtil { 18 | 19 | private SSLParamUtil() { 20 | throw new IllegalStateException(); 21 | } 22 | 23 | /** 24 | * Put certificate directory path to param. 25 | * 26 | * @param param param to store certificate directory 27 | * @param certDirectory certificate directory 28 | * @return changed specified param. 29 | */ 30 | public static Map putSslDirectory( 31 | @NotNull Map param, 32 | @NotNull final String certDirectory 33 | ) { 34 | final HashMap result = new HashMap(param); 35 | result.put(AWSCommonParams.SSL_CERT_DIRECTORY_PARAM, certDirectory); 36 | return result; 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/publish/presigned/util/CloseableS3SignedUrlUploadPool.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish.presigned.util; 2 | 3 | import java.util.concurrent.Callable; 4 | import java.util.concurrent.ExecutorService; 5 | import java.util.concurrent.Future; 6 | import jetbrains.buildServer.util.executors.ExecutorsFactory; 7 | 8 | public class CloseableS3SignedUrlUploadPool implements AutoCloseable { 9 | private final ExecutorService myExecutor; 10 | 11 | public CloseableS3SignedUrlUploadPool(final int parallelism) { 12 | myExecutor = ExecutorsFactory.newFixedExecutor("S3SignedUrlUpload", parallelism); 13 | } 14 | 15 | @Override 16 | public void close() { 17 | myExecutor.shutdown(); 18 | } 19 | 20 | public Future submit(Callable task) { 21 | return myExecutor.submit(task); 22 | } 23 | 24 | public void shutdownNow() { 25 | myExecutor.shutdownNow(); 26 | } 27 | 28 | public boolean isShutdown() { 29 | return myExecutor.isShutdown(); 30 | } 31 | 32 | public boolean isTerminated() { 33 | return myExecutor.isTerminated(); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /s3-artifact-storage-agent/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'io.github.rodm.teamcity-agent' version "1.5.2" 3 | } 4 | 5 | teamcity { 6 | version = teamcityVersion 7 | allowSnapshotVersions = true 8 | agent { 9 | descriptor = project.file('teamcity-plugin.xml') 10 | } 11 | } 12 | 13 | dependencies { 14 | compileOnly("org.jetbrains.teamcity:common-impl:${teamcityVersion}") 15 | implementation project(':s3-artifact-storage-common') 16 | implementation project(':lens-integration') 17 | provided ("org.jetbrains.teamcity:agent-api:${teamcityVersion}"){ 18 | exclude group: 'org.jetbrains.teamcity', module: 'common-api' 19 | } 20 | 21 | testImplementation "org.testng:testng:6.8.21" 22 | testImplementation "org.mockito:mockito-core:3.9.0" 23 | testImplementation "org.jetbrains.teamcity:tests-support:${teamcityVersion}" 24 | testImplementation 'org.testcontainers:testcontainers:1.20.4' 25 | testImplementation("org.jetbrains.teamcity.plugins:aws-core-common:$awsCoreVersion") { 26 | changing = true 27 | } 28 | } 29 | 30 | agentPlugin.version = null 31 | agentPlugin.baseName = projectIds.artifact + '-agent' 32 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/transport/PresignedUrlPartDto.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.transport; 2 | 3 | import javax.xml.bind.annotation.XmlAccessType; 4 | import javax.xml.bind.annotation.XmlAccessorType; 5 | import javax.xml.bind.annotation.XmlRootElement; 6 | import jetbrains.buildServer.Used; 7 | import org.jetbrains.annotations.NotNull; 8 | 9 | @XmlRootElement(name = "presignedUrlPart") 10 | @XmlAccessorType(XmlAccessType.PROPERTY) 11 | public class PresignedUrlPartDto { 12 | private String url; 13 | private int partNumber; 14 | 15 | @Used("serialization") 16 | public PresignedUrlPartDto() { 17 | } 18 | 19 | public PresignedUrlPartDto(@NotNull final String url, final int partNumber) { 20 | this.url = url; 21 | this.partNumber = partNumber; 22 | } 23 | 24 | @NotNull 25 | public String getUrl() { 26 | return url; 27 | } 28 | 29 | public void setUrl(@NotNull String url) { 30 | this.url = url; 31 | } 32 | 33 | public int getPartNumber() { 34 | return partNumber; 35 | } 36 | 37 | public void setPartNumber(int partNumber) { 38 | this.partNumber = partNumber; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /s3-artifact-storage-server/src/main/java/jetbrains/buildServer/artifacts/s3/orphans/BuildEntry.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.orphans; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.Objects; 6 | 7 | class BuildEntry { 8 | private final String myPath; 9 | private final String myId; 10 | 11 | BuildEntry(@NotNull String path, @NotNull String id) { 12 | myPath = path; 13 | myId = id; 14 | } 15 | 16 | @NotNull 17 | String getId() { 18 | return myId; 19 | } 20 | 21 | @NotNull 22 | String getPath() { 23 | return myPath; 24 | } 25 | 26 | @Override 27 | public boolean equals(Object o) { 28 | if (this == o) return true; 29 | if (o == null || getClass() != o.getClass()) return false; 30 | BuildEntry that = (BuildEntry) o; 31 | return Objects.equals(myId, that.myId) && Objects.equals(myPath, that.myPath); 32 | } 33 | 34 | @Override 35 | public int hashCode() { 36 | return Objects.hash(myPath, myId); 37 | } 38 | 39 | @Override 40 | public String toString() { 41 | return "BuildEntry{" + 42 | "myPath='" + myPath + '\'' + 43 | ", myId=" + myId + 44 | '}'; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/publish/presigned/util/RepeatableFilePartRequestEntityApacheLegacy.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish.presigned.util; 2 | 3 | import java.io.IOException; 4 | import java.io.OutputStream; 5 | import org.apache.commons.httpclient.methods.RequestEntity; 6 | import org.jetbrains.annotations.NotNull; 7 | 8 | public class RepeatableFilePartRequestEntityApacheLegacy implements RequestEntity { 9 | @NotNull 10 | private final FilePart myFilePart; 11 | @NotNull private final String myContentType; 12 | 13 | public RepeatableFilePartRequestEntityApacheLegacy(@NotNull FilePart filePart, @NotNull String contentType) { 14 | myFilePart = filePart; 15 | myContentType = contentType; 16 | } 17 | 18 | @NotNull 19 | @Override 20 | public String getContentType() { 21 | return myContentType; 22 | } 23 | 24 | @Override 25 | public boolean isRepeatable() { 26 | return true; 27 | } 28 | 29 | @Override 30 | public long getContentLength() { 31 | return myFilePart.getLength(); 32 | } 33 | 34 | @Override 35 | public void writeRequest(OutputStream outputStream) throws IOException { 36 | myFilePart.write(outputStream); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/App/MultipartUpload/components/Threshold.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | FormInput, 3 | FormRow, 4 | LabelWithHelp, 5 | } from '@jetbrains-internal/tcci-react-ui-components'; 6 | import { React } from '@jetbrains/teamcity-api'; 7 | import { useFormContext } from 'react-hook-form'; 8 | 9 | import { FormFields } from '../../appConstants'; 10 | 11 | // const multipartUploadUrl = 'https://www.jetbrains.com/help/teamcity/2022.10/?Configuring+Artifacts+Storage#multipartUpload'; 12 | 13 | function TresholdLabel() { 14 | return ( 15 | 21 | ); 22 | } 23 | 24 | export default function Threshold() { 25 | const { control } = useFormContext(); 26 | return ( 27 | } 29 | labelFor={FormFields.CONNECTION_MULTIPART_THRESHOLD} 30 | > 31 | 36 | 37 | ); 38 | } 39 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/publish/presigned/util/S3MultipartUploadFileSplitter.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish.presigned.util; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | import org.jetbrains.annotations.NotNull; 8 | 9 | public class S3MultipartUploadFileSplitter { 10 | 11 | private final long myChunkSizeInBytes; 12 | 13 | public S3MultipartUploadFileSplitter(long chunkSizeInBytes) { 14 | myChunkSizeInBytes = chunkSizeInBytes; 15 | } 16 | 17 | @NotNull 18 | public List getFileParts(@NotNull File file, int nParts, boolean checkConsistency) throws IOException { 19 | final List results = new ArrayList<>(); 20 | for (int partIndex = 0; partIndex < nParts; partIndex++) { 21 | final long start = partIndex * myChunkSizeInBytes; 22 | final long contentLength = Math.min(myChunkSizeInBytes, file.length() - myChunkSizeInBytes * partIndex); 23 | final FilePart part = new FilePart(file, start, contentLength, partIndex + 1); 24 | if (checkConsistency) { 25 | part.calculateDigest(); 26 | } 27 | results.add(part); 28 | } 29 | return results; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/App/Storage/components/StorageId.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | FormInput, 3 | FormRow, 4 | LabelWithHelp, 5 | } from '@jetbrains-internal/tcci-react-ui-components'; 6 | import { React } from '@jetbrains/teamcity-api'; 7 | 8 | import { useFormContext } from 'react-hook-form'; 9 | 10 | import { FormFields } from '../../appConstants'; 11 | import { useAppContext } from '../../../contexts/AppContext'; 12 | 13 | function StorageIdLabel() { 14 | return ( 15 | 21 | ); 22 | } 23 | export default function StorageId() { 24 | const { isNewStorage } = useAppContext(); 25 | const { control } = useFormContext(); 26 | 27 | return ( 28 | } 30 | star 31 | labelFor={`${FormFields.STORAGE_ID}_key`} 32 | > 33 | 40 | 41 | ); 42 | } 43 | -------------------------------------------------------------------------------- /lens-integration/src/main/java/jetbrains/buildServer/artifacts/s3/lens/integration/dto/UploadInfoEvent.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.lens.integration.dto; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | 5 | public class UploadInfoEvent { 6 | @JsonProperty("build.artifacts.upload.duration") 7 | private long duration; 8 | @JsonProperty("build.artifacts.count") 9 | private long numberOfFiles; 10 | @JsonProperty("build.artifacts.size") 11 | private long totalSize; 12 | 13 | public UploadInfoEvent() { 14 | } 15 | 16 | public UploadInfoEvent(long duration, long numberOfFiles, long totalSize) { 17 | this.duration = duration; 18 | this.numberOfFiles = numberOfFiles; 19 | this.totalSize = totalSize; 20 | } 21 | 22 | public long getDuration() { 23 | return duration; 24 | } 25 | 26 | public void setDuration(long duration) { 27 | this.duration = duration; 28 | } 29 | 30 | public long getNumberOfFiles() { 31 | return numberOfFiles; 32 | } 33 | 34 | public void setNumberOfFiles(long numberOfFiles) { 35 | this.numberOfFiles = numberOfFiles; 36 | } 37 | 38 | public long getTotalSize() { 39 | return totalSize; 40 | } 41 | 42 | public void setTotalSize(long totalSize) { 43 | this.totalSize = totalSize; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /s3-artifact-storage-server/src/main/java/jetbrains/buildServer/artifacts/s3/web/S3CompatibleArtifactDownloadProcessor.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.web; 2 | 3 | import jetbrains.buildServer.ExtensionsProvider; 4 | import jetbrains.buildServer.artifacts.s3.S3Constants; 5 | import jetbrains.buildServer.artifacts.s3.cloudfront.CloudFrontEnabledPresignedUrlProvider; 6 | import jetbrains.buildServer.serverSide.ProjectManagerEx; 7 | import jetbrains.buildServer.web.ContentSecurityPolicyConfig; 8 | import org.jetbrains.annotations.NotNull; 9 | 10 | public class S3CompatibleArtifactDownloadProcessor extends S3ArtifactDownloadProcessor { 11 | 12 | public S3CompatibleArtifactDownloadProcessor(@NotNull CloudFrontEnabledPresignedUrlProvider preSignedUrlProvider, 13 | @NotNull ExtensionsProvider extensionsProvider, 14 | @NotNull ContentSecurityPolicyConfig contentSecurityPolicyConfig, 15 | @NotNull ProjectManagerEx projectManager) { 16 | super(preSignedUrlProvider, extensionsProvider, contentSecurityPolicyConfig, projectManager); 17 | } 18 | 19 | @NotNull 20 | @Override 21 | public String getType() { 22 | return S3Constants.S3_COMPATIBLE_STORAGE_TYPE; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/publish/errors/HttpResponseAdapter.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish.errors; 2 | 3 | import java.io.IOException; 4 | import jetbrains.buildServer.util.HTTPRequestBuilder; 5 | import jetbrains.buildServer.util.StringUtil; 6 | import org.jetbrains.annotations.NotNull; 7 | import org.jetbrains.annotations.Nullable; 8 | 9 | public class HttpResponseAdapter implements ResponseAdapter { 10 | @NotNull 11 | private final HTTPRequestBuilder.Response myDelegate; 12 | @Nullable 13 | private final String myResponse; 14 | 15 | public HttpResponseAdapter(@NotNull final HTTPRequestBuilder.Response response) throws IOException { 16 | myDelegate = response; 17 | myResponse = response.getBodyAsString(); 18 | } 19 | 20 | @Override 21 | public int getStatusCode() { 22 | return myDelegate.getStatusCode(); 23 | } 24 | 25 | @Nullable 26 | @Override 27 | public String getResponse() { 28 | return myResponse; 29 | } 30 | 31 | @Nullable 32 | @Override 33 | public String getHeader(@NotNull final String header) { 34 | final String responseHeader = myDelegate.getHeader(header); 35 | if (StringUtil.isEmpty(responseHeader)) { 36 | return null; 37 | } else { 38 | return responseHeader; 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/exceptions/FileUploadFailedException.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.exceptions; 2 | 3 | import jetbrains.buildServer.util.ExceptionUtil; 4 | import jetbrains.buildServer.util.retry.RecoverableException; 5 | import org.jetbrains.annotations.NotNull; 6 | 7 | public class FileUploadFailedException extends RecoverableException { 8 | private final boolean myRecoverable; 9 | 10 | public FileUploadFailedException(@NotNull final String msg, @NotNull final RecoverableException e) { 11 | super(msg); 12 | myRecoverable = e.isRecoverable(); 13 | } 14 | 15 | public FileUploadFailedException(@NotNull final String msg, final boolean recoverable) { 16 | super(msg); 17 | myRecoverable = recoverable; 18 | } 19 | 20 | public FileUploadFailedException(@NotNull final String msg, final boolean recoverable, @NotNull final Throwable cause) { 21 | super(msg, cause); 22 | RecoverableException recoverableException = ExceptionUtil.getCause(cause, RecoverableException.class); 23 | if (recoverableException != null) { 24 | myRecoverable = recoverableException.isRecoverable(); 25 | } else { 26 | myRecoverable = recoverable; 27 | } 28 | } 29 | 30 | @Override 31 | public boolean isRecoverable() { 32 | return myRecoverable; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /s3-artifact-storage-server/src/main/java/jetbrains/buildServer/artifacts/s3/util/ParamUtil.java: -------------------------------------------------------------------------------- 1 | 2 | 3 | package jetbrains.buildServer.artifacts.s3.util; 4 | 5 | import java.util.Map; 6 | import jetbrains.buildServer.artifacts.s3.SSLParamUtil; 7 | import jetbrains.buildServer.serverSide.SProject; 8 | import jetbrains.buildServer.serverSide.ServerPaths; 9 | import jetbrains.buildServer.serverSide.TrustedCertificatesDirectory; 10 | import org.jetbrains.annotations.NotNull; 11 | 12 | /** 13 | * Util class for some work with parameters. 14 | * 15 | * @author Mikhail Khorkov 16 | * @since 2018.1 17 | */ 18 | public class ParamUtil { 19 | 20 | private ParamUtil() { 21 | throw new IllegalStateException(); 22 | } 23 | 24 | /** 25 | * Put certificate directory path to param. 26 | * 27 | * @param serverPaths server paths 28 | * @param param param to store certificate directory 29 | * @return the new param map. 30 | */ 31 | @NotNull 32 | public static Map putSslValues( 33 | @NotNull final ServerPaths serverPaths, 34 | @NotNull Map param 35 | ) { 36 | final String certDirectory = TrustedCertificatesDirectory.getCertificateDirectoryForProject( 37 | serverPaths.getProjectsDir().getPath(), SProject.ROOT_PROJECT_ID); 38 | return SSLParamUtil.putSslDirectory(param, certDirectory); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/App/components/Bucket.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | FormRow, 3 | FormSelect, 4 | } from '@jetbrains-internal/tcci-react-ui-components'; 5 | import { React } from '@jetbrains/teamcity-api'; 6 | import { useFormContext } from 'react-hook-form'; 7 | 8 | import { FormFields } from '../appConstants'; 9 | import { IFormInput } from '../../types'; 10 | import { useBucketsContext } from '../../contexts/BucketsContext'; 11 | import useCanLoadBucketInfoData from '../../hooks/useCanLoadBucketInfoData'; 12 | 13 | export default function Bucket() { 14 | const canLoadBucketInfo = useCanLoadBucketInfoData(); 15 | const isDisabled = !canLoadBucketInfo; 16 | const { control } = useFormContext(); 17 | const { 18 | bucketOptions: buckets, 19 | isLoading: bucketsListLoading, 20 | reloadBucketOptions: reloadBuckets, 21 | } = useBucketsContext(); 22 | 23 | return ( 24 | 25 | 36 | 37 | ); 38 | } 39 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/Utilities/fetchBucketNames.ts: -------------------------------------------------------------------------------- 1 | import { ResponseErrors } from '@jetbrains-internal/tcci-react-ui-components/dist/types'; 2 | 3 | import { FetchResourceIds } from '../App/appConstants'; 4 | 5 | import { Config, IFormInput } from '../types'; 6 | 7 | import { post } from './fetchHelper'; 8 | import { serializeParameters } from './parametersUtils'; 9 | import { 10 | displayErrorsFromResponseIfAny, 11 | parseResourceListFromResponse, 12 | } from './responseParser'; 13 | 14 | export type LoadBucketListResponse = { 15 | bucketNames: string[] | null; 16 | errors: ResponseErrors | null; 17 | }; 18 | 19 | export async function loadBucketList( 20 | config: Config, 21 | data: IFormInput 22 | ): Promise { 23 | const parameters = { 24 | ...serializeParameters(data, config), 25 | resource: FetchResourceIds.BUCKETS, 26 | }; 27 | 28 | return await post(config.containersPath, parameters).then((resp) => { 29 | const response = window.$j(resp); 30 | const errors: ResponseErrors | null = 31 | displayErrorsFromResponseIfAny(response); 32 | if (errors) { 33 | return { bucketNames: null, errors }; 34 | } 35 | 36 | const bucketNames = parseResourceListFromResponse( 37 | response, 38 | 'buckets:eq(0) bucket' 39 | ).map((n) => n.textContent!); 40 | 41 | return { bucketNames, errors: null }; 42 | }); 43 | } 44 | -------------------------------------------------------------------------------- /s3-artifact-storage-agent/src/main/resources/META-INF/build-agent-plugin-s3-storage.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /s3-artifact-storage-server/src/main/java/jetbrains/buildServer/artifacts/s3/util/RegionSortPriority.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.util; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.Arrays; 6 | import java.util.Map; 7 | import java.util.function.Function; 8 | import java.util.stream.Collectors; 9 | 10 | public enum RegionSortPriority { 11 | US("us", 10), 12 | EU("eu", 20), 13 | CA("ca", 30), 14 | AP("ap", 40), 15 | SA("sa", 50), 16 | DEFAULT("default", 55), 17 | CN("cn", 60), 18 | US_ISO("us-iso", 70), 19 | US_ISOB("us-isob", 80), 20 | US_GOV("us-gov", 90); 21 | 22 | private static final Map PREFIX_TO_REGION_SORT_PRIORITY = 23 | Arrays.stream(RegionSortPriority.values()) 24 | .collect(Collectors.toMap(k -> k.myPrefix, Function.identity())); 25 | 26 | private final String myPrefix; 27 | private final int myPriority; 28 | 29 | RegionSortPriority(@NotNull final String prefix, final int priority) { 30 | myPrefix = prefix; 31 | myPriority = priority; 32 | } 33 | 34 | public String getPrefix() { 35 | return myPrefix; 36 | } 37 | 38 | public int getPriority() { 39 | return myPriority; 40 | } 41 | 42 | public static RegionSortPriority getFromPrefix(@NotNull final String prefix) { 43 | return PREFIX_TO_REGION_SORT_PRIORITY.getOrDefault(prefix, RegionSortPriority.DEFAULT); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/publish/logger/CompositeS3UploadLogger.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish.logger; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | import org.jetbrains.annotations.NotNull; 6 | 7 | public class CompositeS3UploadLogger implements S3UploadLogger { 8 | @NotNull 9 | private final List myLoggers; 10 | 11 | private CompositeS3UploadLogger(@NotNull final List loggers) { 12 | myLoggers = loggers; 13 | } 14 | 15 | @NotNull 16 | public static S3UploadLogger compose(@NotNull final S3UploadLogger... logger) { 17 | return new CompositeS3UploadLogger(Arrays.asList(logger)); 18 | } 19 | 20 | @Override 21 | public void debug(@NotNull String message) { 22 | for (S3UploadLogger logger : myLoggers) { 23 | logger.debug(message); 24 | } 25 | } 26 | 27 | @Override 28 | public void info(@NotNull String message) { 29 | for (S3UploadLogger logger : myLoggers) { 30 | logger.info(message); 31 | } 32 | } 33 | 34 | @Override 35 | public void warn(@NotNull String message) { 36 | for (S3UploadLogger logger : myLoggers) { 37 | logger.warn(message); 38 | } 39 | } 40 | 41 | @Override 42 | public void error(@NotNull String message) { 43 | for (S3UploadLogger logger : myLoggers) { 44 | logger.error(message); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /s3-artifact-storage-common/src/main/java/jetbrains/buildServer/artifacts/s3/S3ArtifactUtil.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import jetbrains.buildServer.util.StringUtil; 6 | import org.jetbrains.annotations.NotNull; 7 | import org.jetbrains.annotations.Nullable; 8 | 9 | public class S3ArtifactUtil { 10 | 11 | @NotNull 12 | public static String getPathPrefix(@Nullable final String customPrefix, @NotNull final String projectId, @NotNull final String buildTypeId, final long buildId) { 13 | final List pathSegments = new ArrayList<>(); 14 | if (!StringUtil.isEmptyOrSpaces(customPrefix)) { 15 | pathSegments.add(customPrefix); 16 | } 17 | pathSegments.add(projectId); 18 | pathSegments.add(buildTypeId); 19 | pathSegments.add(Long.toString(buildId)); 20 | return StringUtil.join("/", pathSegments) + "/"; 21 | } 22 | 23 | public static boolean matchBuildId(String prefix, String key, long buildId) { 24 | int idx; 25 | if (StringUtil.isEmpty(prefix)) { 26 | idx = 0; 27 | } else if (key.startsWith(prefix + "/")) { 28 | idx = prefix.length() + 1; 29 | } else { 30 | return false; 31 | } 32 | idx = key.indexOf('/', idx); 33 | if (idx < 0) return false; 34 | idx = key.indexOf('/', idx + 1); 35 | return idx > 0 && key.startsWith(buildId + "/", idx + 1); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/publish/presigned/util/RepeatableFilePartRequestEntityApache43.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish.presigned.util; 2 | 3 | import java.io.IOException; 4 | import java.io.InputStream; 5 | import java.io.OutputStream; 6 | import java.io.UnsupportedEncodingException; 7 | import org.apache.http.entity.AbstractHttpEntity; 8 | import org.jetbrains.annotations.NotNull; 9 | 10 | public class RepeatableFilePartRequestEntityApache43 extends AbstractHttpEntity { 11 | @NotNull 12 | private final FilePart myFilePart; 13 | 14 | public RepeatableFilePartRequestEntityApache43(@NotNull FilePart filePart, @NotNull String contentType) { 15 | myFilePart = filePart; 16 | setContentType(contentType); 17 | } 18 | 19 | @Override 20 | public boolean isRepeatable() { 21 | return true; 22 | } 23 | 24 | @Override 25 | public long getContentLength() { 26 | return myFilePart.getLength(); 27 | } 28 | 29 | @Override 30 | public InputStream getContent() throws IOException, UnsupportedOperationException { 31 | throw new UnsupportedEncodingException("File part request doet not allow to get stream to content"); 32 | } 33 | 34 | @Override 35 | public void writeTo(OutputStream outputStream) throws IOException { 36 | myFilePart.write(outputStream); 37 | } 38 | 39 | @Override 40 | public boolean isStreaming() { 41 | return false; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/transport/MultipartUploadAbortRequestDto.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.transport; 2 | 3 | import javax.xml.bind.annotation.XmlAccessType; 4 | import javax.xml.bind.annotation.XmlAccessorType; 5 | import javax.xml.bind.annotation.XmlRootElement; 6 | import jetbrains.buildServer.Used; 7 | import org.jetbrains.annotations.NotNull; 8 | 9 | @XmlRootElement(name = "multipartUploadAbortRequest") 10 | @XmlAccessorType(XmlAccessType.PROPERTY) 11 | public class MultipartUploadAbortRequestDto { 12 | private String objectKey; 13 | private String uploadId; 14 | 15 | @Used("serialization") 16 | public MultipartUploadAbortRequestDto() { 17 | } 18 | 19 | @Used("clients") 20 | public MultipartUploadAbortRequestDto(@NotNull final String objectKey) { 21 | this.objectKey = objectKey; 22 | } 23 | 24 | @Used("clients") 25 | public MultipartUploadAbortRequestDto(@NotNull final String objectKey, @NotNull final String uploadId) { 26 | this.objectKey = objectKey; 27 | this.uploadId = uploadId; 28 | } 29 | 30 | public String getObjectKey() { 31 | return objectKey; 32 | } 33 | 34 | public void setObjectKey(String objectKey) { 35 | this.objectKey = objectKey; 36 | } 37 | 38 | public String getUploadId() { 39 | return uploadId; 40 | } 41 | 42 | public void setUploadId(String uploadId) { 43 | this.uploadId = uploadId; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/App/Storage/components/StorageName.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | FormInput, 3 | FormRow, 4 | } from '@jetbrains-internal/tcci-react-ui-components'; 5 | import { React } from '@jetbrains/teamcity-api'; 6 | import { useFormContext } from 'react-hook-form'; 7 | import DOMPurify from 'dompurify'; 8 | 9 | import { FormFields } from '../../appConstants'; 10 | 11 | function diffStrings(original: string, cleaned: string) { 12 | const unique = new Set(cleaned); 13 | let diff = ''; 14 | for (let i = 0; i < original.length; i++) { 15 | if (!unique.has(original[i])) { 16 | diff += original[i]; 17 | } 18 | } 19 | return diff; 20 | } 21 | export default function StorageName() { 22 | const { control } = useFormContext(); 23 | const sanitizeValidation = (value: string) => { 24 | const cleanValue = DOMPurify.sanitize(value); 25 | return cleanValue === value 26 | ? true 27 | : `Invalid input: ${diffStrings(value, cleanValue) || value}`; 28 | // if diff is empty and cleanValue is not empty, then the original value is some sort of 29 | // acceptable tag that wasn't closed properly; we show it as an error. 30 | }; 31 | 32 | return ( 33 | 34 | 39 | 40 | ); 41 | } 42 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/Utilities/fetchBucketLocation.ts: -------------------------------------------------------------------------------- 1 | import { ResponseErrors } from '@jetbrains-internal/tcci-react-ui-components/dist/types'; 2 | 3 | import { Config, IFormInput } from '../types'; 4 | import { FetchResourceIds } from '../App/appConstants'; 5 | 6 | import { 7 | displayErrorsFromResponseIfAny, 8 | parseResourceListFromResponse, 9 | } from './responseParser'; 10 | import { serializeParameters } from './parametersUtils'; 11 | import { post } from './fetchHelper'; 12 | 13 | export type FetchBucketLoactionResponse = { 14 | location: string | null; 15 | errors: ResponseErrors | null; 16 | }; 17 | 18 | export async function fetchBucketLocation( 19 | config: Config, 20 | data: IFormInput 21 | ): Promise { 22 | const parameters = { 23 | ...serializeParameters(data, config), 24 | resource: FetchResourceIds.BUCKET_LOCATION, 25 | }; 26 | 27 | return await post(config.containersPath, parameters).then((resp) => { 28 | const response = window.$j(resp); 29 | const errors: ResponseErrors | null = 30 | displayErrorsFromResponseIfAny(response); 31 | if (errors) { 32 | return { location: null, errors }; 33 | } 34 | 35 | const location = 36 | parseResourceListFromResponse(response, 'bucket:eq(0)') 37 | .map((it) => window.$j(it)) 38 | .map((n) => n.attr('location'))?.[0] ?? null; 39 | 40 | return { location, errors: null }; 41 | }); 42 | } 43 | -------------------------------------------------------------------------------- /lens-integration/src/main/java/jetbrains/buildServer/artifacts/s3/lens/integration/JsonEntityProducer.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.lens.integration; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import java.io.UnsupportedEncodingException; 6 | import java.nio.charset.StandardCharsets; 7 | import jetbrains.buildServer.util.http.EntityProducer; 8 | import org.apache.commons.httpclient.methods.RequestEntity; 9 | import org.apache.commons.httpclient.methods.StringRequestEntity; 10 | import org.apache.http.HttpEntity; 11 | import org.apache.http.entity.ContentType; 12 | import org.apache.http.entity.StringEntity; 13 | 14 | class JsonEntityProducer implements EntityProducer { 15 | private final static ObjectMapper OBJECT_MAPPER = new ObjectMapper(); 16 | private final String myJson; 17 | 18 | public JsonEntityProducer(Object event) throws JsonProcessingException { 19 | myJson = OBJECT_MAPPER.writeValueAsString(event); 20 | } 21 | 22 | @Override 23 | public HttpEntity entity4() { 24 | return new StringEntity(myJson, ContentType.APPLICATION_JSON); 25 | } 26 | 27 | @Override 28 | public RequestEntity entity3() { 29 | try { 30 | return new StringRequestEntity(myJson, ContentType.APPLICATION_JSON.getMimeType(), StandardCharsets.UTF_8.name()); 31 | } catch (UnsupportedEncodingException e) { 32 | throw new RuntimeException(e); 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/Utilities/fetchS3TransferAccelerationAvailability.ts: -------------------------------------------------------------------------------- 1 | import { ResponseErrors } from '@jetbrains-internal/tcci-react-ui-components/dist/types'; 2 | 3 | import { Config, IFormInput } from '../types'; 4 | import { FetchResourceIds } from '../App/appConstants'; 5 | 6 | import { serializeParameters } from './parametersUtils'; 7 | import { post } from './fetchHelper'; 8 | import { parseErrorsFromResponse, parseResponse } from './responseParser'; 9 | 10 | type FetchS3TransferAccelerationAvailabilityResponse = { 11 | isAvailable: boolean | null; 12 | errors: ResponseErrors | null; 13 | }; 14 | export async function fetchS3TransferAccelerationAvailability( 15 | config: Config, 16 | data: IFormInput 17 | ): Promise { 18 | const parameters = { 19 | ...serializeParameters(data, config), 20 | resource: FetchResourceIds.S3_TRANSFER_ACCELERATION_AVAILABILITY, 21 | }; 22 | 23 | return await post(config.containersPath, parameters).then((resp) => { 24 | const response = new DOMParser().parseFromString(resp, 'text/xml'); 25 | const errors: ResponseErrors | null = parseErrorsFromResponse(response); 26 | if (errors) { 27 | return { isAvailable: null, errors }; 28 | } 29 | 30 | const isAvailable = 31 | parseResponse(response, 's3Acceleration')[0]?.getAttribute( 32 | 'accelerationStatus' 33 | ) === 'Enabled'; 34 | return { isAvailable, errors: null }; 35 | }); 36 | } 37 | -------------------------------------------------------------------------------- /lens-integration/src/test/java/jetbrains/buildServer/artifacts/s3/lens/integration/TestJsonGeneration.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.lens.integration; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import jetbrains.buildServer.artifacts.s3.lens.integration.dto.UploadFileEvent; 6 | import org.testng.Assert; 7 | import org.testng.annotations.Test; 8 | 9 | @Test 10 | public class TestJsonGeneration { 11 | private static final String TEST_JSON_STRING = 12 | "{\"build.artifacts.object.upload.result\":\"successful\",\"build.artifacts.object.key\":\"file.zip\",\"build.artifacts.object.size\":123456789,\"build.artifacts.object.chunk_count\":1,\"build.artifacts.object.chunk_size\":123456789,\"build.artifacts.object.upload.duration\":123456789,\"build.artifacts.object.upload_retry_count\":0}"; 13 | 14 | @Test 15 | public void testDtoToJson() throws JsonProcessingException { 16 | UploadFileEvent event = new UploadFileEvent(); 17 | event.setObjectKey("file.zip"); 18 | event.setFileSize(123456789L); 19 | event.setDuration(123456789L); 20 | event.setNumberOfParts(1); 21 | event.setChunkSize(123456789L); 22 | event.setRestartCount(0); 23 | event.setUploadResult("successful"); 24 | 25 | ObjectMapper mapper = new ObjectMapper(); 26 | String json = mapper.writeValueAsString(event); 27 | System.out.println(json); 28 | Assert.assertEquals(json, TEST_JSON_STRING); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/S3ClientResourceFetcher.java: -------------------------------------------------------------------------------- 1 | 2 | 3 | package jetbrains.buildServer.artifacts.s3; 4 | 5 | import java.util.Map; 6 | import jetbrains.buildServer.artifacts.s3.serialization.S3XmlSerializerFactory; 7 | import org.jdom.Element; 8 | import org.jetbrains.annotations.NotNull; 9 | 10 | public abstract class S3ClientResourceFetcher { 11 | 12 | @NotNull 13 | public String fetchAsXml(final Map parameters, @NotNull final String projectId) throws Exception { 14 | final T dto = fetchDto(parameters, projectId); 15 | return S3XmlSerializerFactory.getInstance().serialize(dto); 16 | } 17 | 18 | @NotNull 19 | public Element fetchAsElement(final Map parameters, @NotNull final String projectId) throws Exception { 20 | final T dto = fetchDto(parameters, projectId); 21 | return S3XmlSerializerFactory.getInstance().serializeAsElement(dto); 22 | } 23 | 24 | public Element fetchCurrentValueAsElement(final Map parameters, @NotNull final String projectId) throws Exception{ 25 | final T dto = fetchCurrentValue(parameters, projectId); 26 | return S3XmlSerializerFactory.getInstance().serializeAsElement(dto); 27 | } 28 | 29 | protected abstract T fetchCurrentValue(final Map parameters, @NotNull final String projectId) throws Exception; 30 | 31 | protected abstract T fetchDto(final Map parameters, @NotNull final String projectId) throws Exception; 32 | } 33 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/S3PresignedUrlProvider.java: -------------------------------------------------------------------------------- 1 | 2 | 3 | package jetbrains.buildServer.artifacts.s3; 4 | 5 | import java.io.IOException; 6 | import java.util.Map; 7 | 8 | import org.jetbrains.annotations.NotNull; 9 | import org.jetbrains.annotations.Nullable; 10 | import software.amazon.awssdk.http.SdkHttpMethod; 11 | 12 | /** 13 | * Created by Evgeniy Koshkin (evgeniy.koshkin@jetbrains.com) on 19.07.17. 14 | */ 15 | public interface S3PresignedUrlProvider { 16 | 17 | @NotNull 18 | PresignedUrlWithTtl generateDownloadUrl(@NotNull SdkHttpMethod httpMethod, @NotNull String objectKey, @NotNull S3Settings settings) throws IOException; 19 | 20 | @NotNull 21 | String generateUploadUrl(@NotNull String objectKey, @Nullable String digest, @NotNull S3Settings settings) throws IOException; 22 | 23 | @NotNull 24 | String generateUploadUrlForPart(@NotNull String objectKey, @Nullable String digest, int nPart, @NotNull String uploadId, @NotNull S3Settings settings) throws IOException; 25 | 26 | void finishMultipartUpload(@NotNull String uploadId, @NotNull String objectKey, @NotNull S3Settings settings, @Nullable String[] etags, boolean isSuccessful) throws IOException; 27 | 28 | @NotNull 29 | String startMultipartUpload(@NotNull String objectKey, @Nullable String contentType, @NotNull S3Settings settings) throws Exception; 30 | 31 | @NotNull 32 | S3Settings settings(@NotNull Map rawSettings, @NotNull Map projectSettings); 33 | 34 | } 35 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/App/ProtocolSettings/ProtocolSettings.tsx: -------------------------------------------------------------------------------- 1 | import { React } from '@jetbrains/teamcity-api'; 2 | import { 3 | FormCheckbox, 4 | SectionHeader, 5 | } from '@jetbrains-internal/tcci-react-ui-components'; 6 | 7 | import { useFormContext } from 'react-hook-form'; 8 | 9 | import { useEffect } from 'react'; 10 | 11 | import { FormFields } from '../appConstants'; 12 | import { IFormInput } from '../../types'; 13 | 14 | export default function ProtocolSettings() { 15 | const { control, setValue, getValues, watch } = useFormContext(); 16 | const s3TransferAcceleration = watch( 17 | FormFields.CONNECTION_TRANSFER_ACCELERATION_TOGGLE 18 | ); 19 | 20 | useEffect(() => { 21 | if (s3TransferAcceleration) { 22 | setValue(FormFields.CONNECTION_FORCE_VHA_TOGGLE, true); 23 | } 24 | }, [s3TransferAcceleration, setValue]); 25 | 26 | return ( 27 |
28 | {'Protocol settings'} 29 | 38 | 43 |
44 | ); 45 | } 46 | -------------------------------------------------------------------------------- /s3-artifact-storage-server/src/main/java/jetbrains/buildServer/artifacts/s3/orphans/BuildTypeEntry.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.orphans; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.HashSet; 6 | import java.util.Objects; 7 | import java.util.Set; 8 | 9 | class BuildTypeEntry { 10 | private final String myPath; 11 | private final Set myBuildEntries; 12 | private final boolean myOutdated; 13 | 14 | BuildTypeEntry(@NotNull String path, @NotNull Set buildEntries, boolean isOutdated) { 15 | myPath = path; 16 | myBuildEntries = new HashSet<>(buildEntries); 17 | myOutdated = isOutdated; 18 | } 19 | 20 | @NotNull 21 | String getPath() { 22 | return myPath; 23 | } 24 | 25 | @NotNull 26 | Set getBuildEntries() { 27 | return myBuildEntries; 28 | } 29 | 30 | boolean isOutdated() { 31 | return myOutdated; 32 | } 33 | 34 | @Override 35 | public boolean equals(Object o) { 36 | if (this == o) return true; 37 | if (o == null || getClass() != o.getClass()) return false; 38 | BuildTypeEntry that = (BuildTypeEntry) o; 39 | return Objects.equals(myPath, that.myPath); 40 | } 41 | 42 | @Override 43 | public int hashCode() { 44 | return Objects.hashCode(myPath); 45 | } 46 | 47 | @Override 48 | public String toString() { 49 | return "BuildTypeEntry{" + 50 | "myPath='" + myPath + '\'' + 51 | ", myBuildEntries=" + myBuildEntries + 52 | ", myOutdated=" + myOutdated + 53 | '}'; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/publish/presigned/util/DigestUtil.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish.presigned.util; 2 | 3 | import com.intellij.openapi.diagnostic.Logger; 4 | import java.io.ByteArrayOutputStream; 5 | import java.io.IOException; 6 | import java.security.MessageDigest; 7 | import java.security.NoSuchAlgorithmException; 8 | import java.util.List; 9 | import jetbrains.buildServer.artifacts.s3.exceptions.FileUploadFailedException; 10 | import org.apache.commons.codec.DecoderException; 11 | import org.apache.commons.codec.binary.Hex; 12 | import org.apache.commons.codec.digest.DigestUtils; 13 | import org.jetbrains.annotations.NotNull; 14 | 15 | public final class DigestUtil { 16 | @NotNull 17 | private static final Logger LOGGER = Logger.getInstance(DigestUtil.class); 18 | 19 | private DigestUtil() { 20 | } 21 | 22 | public static String multipartDigest(@NotNull final List etags) { 23 | try (final ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { 24 | etags.forEach(etag -> { 25 | try { 26 | outputStream.write(Hex.decodeHex(etag)); 27 | } catch (DecoderException | IOException e) { 28 | throw new FileUploadFailedException("Decode of etag " + etag + " failed", true); 29 | } 30 | }); 31 | return DigestUtils.md5Hex(outputStream.toByteArray()) + "-" + etags.size(); 32 | } catch (IOException e) { 33 | LOGGER.debug("Got exception while closing bytearrayoutputstream", e); 34 | return ""; 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/transport/MultipartUploadCompleteRequestDto.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.transport; 2 | 3 | import java.util.List; 4 | import javax.xml.bind.annotation.XmlAccessType; 5 | import javax.xml.bind.annotation.XmlAccessorType; 6 | import javax.xml.bind.annotation.XmlRootElement; 7 | import jetbrains.buildServer.Used; 8 | import org.jetbrains.annotations.NotNull; 9 | 10 | @XmlRootElement(name = "multipartUploadCompleteRequest") 11 | @XmlAccessorType(XmlAccessType.PROPERTY) 12 | public class MultipartUploadCompleteRequestDto { 13 | private String objectKey; 14 | private String uploadId; 15 | private List etags; 16 | 17 | @Used("serialization") 18 | public MultipartUploadCompleteRequestDto() { 19 | } 20 | 21 | @Used("clients") 22 | public MultipartUploadCompleteRequestDto(@NotNull final String objectKey, @NotNull final String uploadId, @NotNull final List etags) { 23 | this.objectKey = objectKey; 24 | this.uploadId = uploadId; 25 | this.etags = etags; 26 | } 27 | 28 | public String getObjectKey() { 29 | return objectKey; 30 | } 31 | 32 | public void setObjectKey(String objectKey) { 33 | this.objectKey = objectKey; 34 | } 35 | 36 | public String getUploadId() { 37 | return uploadId; 38 | } 39 | 40 | public void setUploadId(String uploadId) { 41 | this.uploadId = uploadId; 42 | } 43 | 44 | public List getEtags() { 45 | return etags; 46 | } 47 | 48 | public void setEtags(List etags) { 49 | this.etags = etags; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/publish/S3FileUploaderFactoryImpl.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish; 2 | 3 | import java.util.function.Supplier; 4 | import jetbrains.buildServer.artifacts.s3.S3Configuration; 5 | import jetbrains.buildServer.artifacts.s3.publish.logger.S3UploadLogger; 6 | import jetbrains.buildServer.artifacts.s3.publish.presigned.upload.PresignedUrlsProviderClient; 7 | import jetbrains.buildServer.artifacts.s3.publish.presigned.upload.S3SignedUrlFileUploader; 8 | import jetbrains.buildServer.serverSide.TeamCityProperties; 9 | import org.jetbrains.annotations.NotNull; 10 | 11 | import static jetbrains.buildServer.artifacts.s3.S3Constants.*; 12 | 13 | public class S3FileUploaderFactoryImpl implements S3FileUploaderFactory { 14 | @Override 15 | public S3FileUploader create(@NotNull final S3Configuration s3Configuration, 16 | @NotNull final S3UploadLogger s3UploadLogger, 17 | @NotNull final Supplier presignedUrlsProviderClientSupplier) { 18 | 19 | if (TeamCityProperties.getBoolean(S3_FORCE_PRESIGNED_URLS, true) && 20 | S3_STORAGE_TYPE.equals(s3Configuration.getSettingsMap().get(TEAMCITY_STORAGE_TYPE_KEY))) { 21 | return new S3SignedUrlFileUploader(s3Configuration, s3UploadLogger, presignedUrlsProviderClientSupplier); 22 | } 23 | 24 | return s3Configuration.isUsePresignedUrls() 25 | ? new S3SignedUrlFileUploader(s3Configuration, s3UploadLogger, presignedUrlsProviderClientSupplier) 26 | : new S3RegularFileUploader(s3Configuration, s3UploadLogger); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/Utilities/fetchPublicKeys.ts: -------------------------------------------------------------------------------- 1 | import { ResponseErrors } from '@jetbrains-internal/tcci-react-ui-components/dist/types'; 2 | 3 | import { FetchResourceIds } from '../App/appConstants'; 4 | 5 | import { Config, IFormInput } from '../types'; 6 | 7 | import { post } from './fetchHelper'; 8 | import { serializeParameters } from './parametersUtils'; 9 | import { 10 | displayErrorsFromResponseIfAny, 11 | parseResourceListFromResponse, 12 | } from './responseParser'; 13 | 14 | export type PublicKeyType = { id: string; name: string }; 15 | 16 | export type LoadPublicKeyListResponse = { 17 | publicKeys: PublicKeyType[] | null; 18 | errors: ResponseErrors | null; 19 | }; 20 | 21 | export async function loadPublicKeyList( 22 | appProps: Config, 23 | allValues: IFormInput 24 | ): Promise { 25 | const parameters = { 26 | ...serializeParameters(allValues, appProps), 27 | resource: FetchResourceIds.PUBLIC_KEYS, 28 | }; 29 | 30 | return await post(appProps.containersPath, parameters).then((resp) => { 31 | const response = window.$j(resp); 32 | 33 | const errors: ResponseErrors | null = 34 | displayErrorsFromResponseIfAny(response); 35 | if (errors) { 36 | return { publicKeys: null, errors }; 37 | } 38 | 39 | const publicKeys = parseResourceListFromResponse( 40 | response, 41 | 'publicKeys:eq(0) publicKey' 42 | ) 43 | .map((n) => window.$j(n)) 44 | .map((g) => { 45 | const id = g.find('id').text(); 46 | const name = g.find('name').text(); 47 | return { id, name }; 48 | }); 49 | 50 | return { publicKeys, errors: null }; 51 | }); 52 | } 53 | -------------------------------------------------------------------------------- /s3-artifact-storage-agent/src/main/java/jetbrains/buildServer/artifacts/s3/download/S3HttpClient.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.download; 2 | 3 | import java.io.IOException; 4 | import jetbrains.buildServer.artifacts.impl.DependencyHttpHelper; 5 | import jetbrains.buildServer.http.HttpUserAgent; 6 | import org.apache.commons.httpclient.HttpClient; 7 | import org.apache.commons.httpclient.HttpMethod; 8 | import org.apache.commons.httpclient.HttpState; 9 | import org.jetbrains.annotations.NotNull; 10 | 11 | public final class S3HttpClient { 12 | @NotNull 13 | private final HttpClient myHttpClient; 14 | @NotNull 15 | private final DependencyHttpHelper myDependencyHttpHelper; 16 | @NotNull 17 | private final String myServerUrl; 18 | 19 | public S3HttpClient(@NotNull HttpClient httpClient, @NotNull DependencyHttpHelper dependencyHttpHelper, @NotNull String serverUrl) { 20 | myHttpClient = httpClient; 21 | myDependencyHttpHelper = dependencyHttpHelper; 22 | myServerUrl = serverUrl; 23 | } 24 | 25 | public int execute(@NotNull HttpMethod request) throws IOException { 26 | HttpUserAgent.addHeader(request).setFollowRedirects(false); 27 | myDependencyHttpHelper.addAdditionalHeaders(request); 28 | 29 | // we need to clear the state because the HttpClient is configured to provide credentials for TC server in the Authorization header 30 | // we don't want to provide them to untrusted parties (see jetbrains.buildServer.artifacts.impl.HttpTransport) 31 | return !request.getURI().getURI().startsWith(myServerUrl) 32 | ? myHttpClient.executeMethod(null, request, new HttpState()) 33 | : myHttpClient.executeMethod(request); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/Utilities/responseParser.ts: -------------------------------------------------------------------------------- 1 | import { ResponseErrors } from '@jetbrains-internal/tcci-react-ui-components/dist/types'; 2 | 3 | export function parseResourceListFromResponse( 4 | response: JQuery, 5 | selector: string 6 | ) { 7 | const list: HTMLElement[] = []; 8 | const elems: JQuery = response.find(selector); 9 | for (let i = 0; i < elems.length; ++i) { 10 | const e: HTMLElement = elems[i]; 11 | list.push(e); 12 | } 13 | return list; 14 | } 15 | 16 | export function displayErrorsFromResponseIfAny(response: JQuery) { 17 | const errors = parseErrors(response); 18 | if (!errors) { 19 | return null; 20 | } 21 | return errors; 22 | } 23 | 24 | export function parseErrors(response: JQuery) { 25 | const errors = response.find('errors:eq(0) error'); 26 | if (!errors.length) { 27 | return null; 28 | } else { 29 | const result: ResponseErrors = {}; 30 | for (let i = 0; i < errors.length; ++i) { 31 | result[errors[i].id] = { message: errors[i].textContent! }; 32 | } 33 | return result; 34 | } 35 | } 36 | 37 | export function parseErrorsFromResponse(response: Document) { 38 | const errors = response.querySelectorAll('errors > error'); 39 | if (!errors.length) { 40 | return null; 41 | } else { 42 | const result: ResponseErrors = {}; 43 | errors.forEach( 44 | (elem) => (result[elem.id] = { message: elem.textContent! }) 45 | ); 46 | return result; 47 | } 48 | } 49 | 50 | export function parseResponse(response: Document, selector: string) { 51 | const result: Element[] = []; 52 | response.querySelectorAll(selector).forEach((elem) => result.push(elem)); 53 | 54 | return result; 55 | } 56 | -------------------------------------------------------------------------------- /s3-artifact-storage-server/src/main/java/jetbrains/buildServer/artifacts/s3/orphans/OrphanedArtifact.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.orphans; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | import org.jetbrains.annotations.Nullable; 5 | 6 | import java.util.Objects; 7 | 8 | public class OrphanedArtifact implements Comparable{ 9 | private final String bucket; 10 | private final String path; 11 | private final String size; 12 | 13 | public OrphanedArtifact(@NotNull String bucket, @NotNull String path, @Nullable String size) { 14 | this.bucket = bucket; 15 | this.path = path; 16 | this.size = size; 17 | } 18 | 19 | public String getPath() { 20 | return path; 21 | } 22 | 23 | public String getSize() { 24 | return size; 25 | } 26 | 27 | public String getBucket() { 28 | return bucket; 29 | } 30 | 31 | @Override 32 | public boolean equals(Object o) { 33 | if (this == o) return true; 34 | if (o == null || getClass() != o.getClass()) return false; 35 | OrphanedArtifact that = (OrphanedArtifact) o; 36 | return Objects.equals(bucket, that.bucket) && Objects.equals(path, that.path) && Objects.equals(size, that.size); 37 | } 38 | 39 | @Override 40 | public int hashCode() { 41 | return Objects.hash(bucket, path, size); 42 | } 43 | 44 | @Override 45 | public String toString() { 46 | return "OrphanedArtifact{" + 47 | "'s3://" + bucket + 48 | "/" + path + 49 | "' (" + size + ")}"; 50 | } 51 | 52 | @Override 53 | public int compareTo(@NotNull OrphanedArtifact o) { 54 | return (this.bucket + this.path).compareTo(o.bucket + o.path); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /s3-artifact-storage-server/src/main/java/jetbrains/buildServer/artifacts/s3/orphans/ProjectEntry.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.orphans; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.Objects; 6 | import java.util.Set; 7 | import java.util.stream.Collectors; 8 | 9 | class ProjectEntry { 10 | 11 | private final String myPath; 12 | private final Set myBuildTypeEntries; 13 | private final boolean myOutdated; 14 | 15 | ProjectEntry(@NotNull String path, @NotNull Set buildTypeEntries) { 16 | myPath = path; 17 | 18 | myBuildTypeEntries = buildTypeEntries; 19 | 20 | long outdatedCount = buildTypeEntries.stream() 21 | .filter(BuildTypeEntry::isOutdated) 22 | .count(); 23 | 24 | myOutdated = myBuildTypeEntries.size() == outdatedCount; 25 | } 26 | 27 | @NotNull 28 | String getPath() { 29 | return myPath; 30 | } 31 | 32 | @NotNull 33 | Set getBuildTypeEntries() { 34 | return myBuildTypeEntries; 35 | } 36 | 37 | boolean isOutdated() { 38 | return myOutdated; 39 | } 40 | 41 | @Override 42 | public boolean equals(Object o) { 43 | if (this == o) return true; 44 | if (o == null || getClass() != o.getClass()) return false; 45 | ProjectEntry that = (ProjectEntry) o; 46 | return Objects.equals(myPath, that.myPath); 47 | } 48 | 49 | @Override 50 | public int hashCode() { 51 | return Objects.hashCode(myPath); 52 | } 53 | 54 | @Override 55 | public String toString() { 56 | return "ProjectEntry{" + 57 | "myPath='" + myPath + '\'' + 58 | ", myBuildTypeEntries=" + myBuildTypeEntries + 59 | ", myOutdated=" + myOutdated + 60 | '}'; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /s3-artifact-storage-server/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'io.github.rodm.teamcity-server' version "1.5.2" 3 | id 'com.github.jk1.tcdeps' version '1.3.1' 4 | } 5 | 6 | teamcity { 7 | version = teamcityVersion 8 | allowSnapshotVersions = true 9 | server { 10 | descriptor = project.file('teamcity-plugin.xml') 11 | tokens = [Version: project.version] 12 | 13 | files { 14 | into("kotlin-dsl") { 15 | from("kotlin-dsl") 16 | } 17 | } 18 | } 19 | } 20 | 21 | dependencies { 22 | provided("com.zaxxer:HikariCP:4.0.3") 23 | implementation(platform("org.jetbrains.teamcity:bom:$teamcityVersion")) 24 | implementation project(':s3-artifact-storage-common') 25 | provided(group: 'org.jetbrains.teamcity.internal', name: 'server') { 26 | exclude group: 'org.apache.httpcomponents', module: 'httpclient' 27 | exclude group: 'org.apache.httpcomponents', module: 'httpcore' 28 | exclude group: 'commons-codec', module: 'commons-codec' 29 | } 30 | implementation "software.amazon.awssdk:iam-policy-builder:2.33.2" 31 | provided "org.jetbrains.teamcity:connections-api" 32 | 33 | implementation "org.bouncycastle:bcprov-jdk18on:1.78" 34 | 35 | testImplementation "org.testcontainers:localstack:1.21.3" 36 | testImplementation "org.mockito:mockito-core:2.1.0" 37 | agent project(path: ':s3-artifact-storage-agent', configuration: 'plugin') 38 | server group: 'commons-codec', name: 'commons-codec', version: '1.13' 39 | testImplementation("org.jetbrains.teamcity.plugins:aws-core-common:$awsCoreVersion") { 40 | changing = true 41 | } 42 | } 43 | 44 | serverPlugin.version = null 45 | serverPlugin.baseName = projectIds.artifact 46 | -------------------------------------------------------------------------------- /s3-artifact-storage-agent/src/main/java/jetbrains/buildServer/artifacts/s3/download/parallel/ParallelDownloadState.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.download.parallel; 2 | 3 | 4 | import java.io.IOException; 5 | import java.util.concurrent.atomic.AtomicBoolean; 6 | import java.util.concurrent.atomic.AtomicReference; 7 | import jetbrains.buildServer.artifacts.FileProgress; 8 | import org.jetbrains.annotations.NotNull; 9 | import org.jetbrains.annotations.Nullable; 10 | 11 | public final class ParallelDownloadState { 12 | @NotNull 13 | private final FileProgress myDownloadProgress; 14 | @NotNull 15 | private final AtomicBoolean myInterruptedFlag; 16 | @NotNull 17 | private final AtomicReference myFirstPartFailure; 18 | 19 | public ParallelDownloadState(@NotNull FileProgress downloadProgress, @NotNull AtomicBoolean interruptedFlag) { 20 | myDownloadProgress = downloadProgress; 21 | myInterruptedFlag = interruptedFlag; 22 | myFirstPartFailure = new AtomicReference<>(null); 23 | } 24 | 25 | public void partFailed(@NotNull FilePart filePart, @NotNull IOException exception) { 26 | myFirstPartFailure.compareAndSet(null, new PartFailure(filePart, exception)); 27 | } 28 | 29 | public boolean hasFailedParts() { 30 | return myFirstPartFailure.get() != null; 31 | } 32 | 33 | @Nullable 34 | public PartFailure getFirstPartFailure() { 35 | return myFirstPartFailure.get(); 36 | } 37 | 38 | public void expectDownloadedBytes(long bytes) { 39 | myDownloadProgress.setExpectedLength(bytes); 40 | } 41 | 42 | public void addDownloadedBytes(long bytes) { 43 | myDownloadProgress.transferred(bytes); 44 | } 45 | 46 | public boolean isInterrupted() { 47 | return myInterruptedFlag.get(); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/App/styles.css: -------------------------------------------------------------------------------- 1 | .App { 2 | margin-top: -10px; 3 | display: flex; 4 | flex-direction: column; 5 | font-family: var(--ring-font-family); 6 | font-size: var(--ring-font-size); 7 | background-color: var(--ring-content-background-color, #fff); 8 | color: var(--ring-text-color, #1f2326); 9 | } 10 | 11 | .formControlButtons { 12 | margin-top: 32px; 13 | } 14 | 15 | .credTypesCustom { 16 | display: flex; 17 | flex-basis: 25%; 18 | flex-direction: column; 19 | } 20 | 21 | .commentary { 22 | inline-size: 400px; 23 | overflow-wrap: break-word; 24 | margin-block-start: 0; 25 | margin-block-end: 0; 26 | 27 | color: #737577; 28 | 29 | font-size: 12px; 30 | line-height: 16px; 31 | } 32 | 33 | .helpButtonShift { 34 | margin-top: 24px 35 | } 36 | 37 | .magicButtonShift { 38 | margin-top: 22px 39 | } 40 | 41 | .uploadButton { 42 | margin-top: 8px; 43 | } 44 | 45 | .cfLoader { 46 | display: inline-flex; 47 | margin-left: 100px; 48 | } 49 | 50 | .convertWarningBox { 51 | margin-top: 12px; 52 | width: 400px; 53 | height: 48px; 54 | padding: 8px 16px; 55 | border-radius: 4px; 56 | box-sizing: border-box; 57 | display: flex; 58 | flex-direction: row; 59 | gap: 8px; 60 | align-items: center; 61 | background: #FAECCD; 62 | } 63 | 64 | .convertWarningBox button { 65 | padding: 0; 66 | height: 16px; 67 | } 68 | 69 | .convertWarningBox button span > span { 70 | color: #0F5B99; 71 | } 72 | 73 | .awsConnections div[class*='label'] { 74 | color: var(--ring-secondary-color); 75 | font-size: var(--ring-font-size-smaller); 76 | line-height: var(--ring-line-height-lowest); 77 | margin-bottom: 0.25rem; 78 | } 79 | 80 | .awsConnections { 81 | margin-top: 1rem; 82 | display: block; 83 | } 84 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/Utilities/fetchCfKeysValidationResult.ts: -------------------------------------------------------------------------------- 1 | import { ResponseErrors } from '@jetbrains-internal/tcci-react-ui-components/dist/types'; 2 | 3 | import { Config, IFormInput } from '../types'; 4 | 5 | import { FetchResourceIds } from '../App/appConstants'; 6 | 7 | import { serializeParameters } from './parametersUtils'; 8 | import { post } from './fetchHelper'; 9 | import { parseErrorsFromResponse, parseResponse } from './responseParser'; 10 | 11 | interface FetchCfKeysValidationResultResponse { 12 | isValid: boolean | null; 13 | errors: ResponseErrors | null; 14 | } 15 | 16 | export async function fetchCfKeysValidationResult( 17 | config: Config, 18 | data: IFormInput 19 | ): Promise { 20 | const parameters = { 21 | ...serializeParameters(data, config), 22 | resource: FetchResourceIds.VALIDATE_CLOUD_FRONT_KEYS, 23 | }; 24 | 25 | return await post(config.containersPath, parameters).then((resp) => { 26 | const response = new DOMParser().parseFromString(resp, 'text/xml'); 27 | const errors: ResponseErrors | null = parseErrorsFromResponse(response); 28 | if (errors) { 29 | return { isValid: null, errors }; 30 | } 31 | 32 | const validationResult = parseResponse( 33 | response, 34 | 'cfKeysValidationResult' 35 | )[0]?.textContent; 36 | const isValid = validationResult === 'OK'; 37 | if (isValid) return { isValid, errors: null }; 38 | else if (validationResult) { 39 | return { 40 | isValid, 41 | errors: { 42 | [FetchResourceIds.VALIDATE_CLOUD_FRONT_KEYS]: { 43 | message: validationResult, 44 | }, 45 | }, 46 | }; 47 | } else { 48 | return { isValid, errors: null }; 49 | } 50 | }); 51 | } 52 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'maven-publish' 2 | 3 | ext { 4 | spaceRepositoryUser = findProperty('space.repository.user') 5 | spaceRepositoryPassword = findProperty('space.repository.password') 6 | } 7 | 8 | dependencies { 9 | compileOnly "org.jetbrains.teamcity:common-api:${teamcityVersion}" 10 | compileOnly "org.jetbrains.teamcity:connections-api:${teamcityVersion}" 11 | compileOnly "eu.bitwalker:UserAgentUtils:1.17" 12 | compileOnly 'commons-httpclient:commons-httpclient:3.1' 13 | compileOnly 'com.google.guava:guava:13.0.1' 14 | 15 | testImplementation "org.testng:testng:6.8.21" 16 | testImplementation "org.jetbrains.teamcity:tests-support:${teamcityVersion}" 17 | testImplementation "org.mockito:mockito-core:3.9.0" 18 | testImplementation("org.jetbrains.teamcity.plugins:aws-core-common:$awsCoreVersion") { 19 | changing = true 20 | } 21 | } 22 | 23 | publishing { 24 | repositories { 25 | if (localAwsRepo) { 26 | maven { 27 | name = "localAwsRepo" 28 | url "file:///${localAwsRepo}" 29 | } 30 | } 31 | maven { 32 | name = "spacePackages" 33 | credentials { 34 | username = spaceRepositoryUser 35 | password = spaceRepositoryPassword 36 | } 37 | url = 'https://packages.jetbrains.team/maven/p/tc/maven' 38 | } 39 | } 40 | 41 | publications { 42 | mavenPrivate(MavenPublication) { 43 | groupId = 'org.jetbrains.teamcity.internal' 44 | artifactId = 'teamcity-s3-sdk' 45 | version = "${teamcityS3SDKVersion}" 46 | from components.java 47 | pom { 48 | name = 'TeamCity S3 SDK' 49 | description = 'Shared S3 SDK Library' 50 | url = 'https://packages.jetbrains.team/maven/p/tc/maven' 51 | } 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /s3-artifact-storage-server/src/main/java/jetbrains/buildServer/filestorage/cloudfront/GuardedCloudFrontPresignedUrlProvider.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.filestorage.cloudfront; 2 | 3 | import java.io.IOException; 4 | import jetbrains.buildServer.artifacts.s3.PresignedUrlWithTtl; 5 | import jetbrains.buildServer.artifacts.s3.cloudfront.CloudFrontPresignedUrlProvider; 6 | import jetbrains.buildServer.artifacts.s3.cloudfront.CloudFrontSettings; 7 | import jetbrains.buildServer.serverSide.IOGuard; 8 | import org.jetbrains.annotations.NotNull; 9 | import org.jetbrains.annotations.Nullable; 10 | 11 | public class GuardedCloudFrontPresignedUrlProvider implements CloudFrontPresignedUrlProvider { 12 | private final CloudFrontPresignedUrlProvider myDelegate; 13 | 14 | public GuardedCloudFrontPresignedUrlProvider(@NotNull CloudFrontPresignedUrlProvider provider) { 15 | myDelegate = provider; 16 | } 17 | 18 | @Nullable 19 | @Override 20 | public PresignedUrlWithTtl generateDownloadUrl(@NotNull String objectKey, 21 | @NotNull CloudFrontSettings settings) throws IOException { 22 | return IOGuard.allowNetworkCall(() -> myDelegate.generateDownloadUrl(objectKey, settings)); 23 | } 24 | 25 | @Nullable 26 | @Override 27 | public String generateUploadUrl(@NotNull String objectKey, 28 | @NotNull CloudFrontSettings settings) throws IOException { 29 | return IOGuard.allowNetworkCall(() -> myDelegate.generateUploadUrl(objectKey, settings)); 30 | } 31 | 32 | @Override 33 | public String generateUploadUrlForPart(@NotNull String objectKey, int nPart, @NotNull String uploadId, @NotNull CloudFrontSettings settings) throws IOException { 34 | return IOGuard.allowNetworkCall(() -> myDelegate.generateUploadUrlForPart(objectKey, nPart, uploadId, settings)); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/publish/errors/TeamCityPresignedUrlsProviderErrorHandler.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish.errors; 2 | 3 | import jetbrains.buildServer.artifacts.s3.S3Constants; 4 | import jetbrains.buildServer.artifacts.s3.publish.presigned.util.HttpClientUtil; 5 | import jetbrains.buildServer.transport.AgentServerSharedErrorMessages; 6 | import org.jetbrains.annotations.NotNull; 7 | 8 | public class TeamCityPresignedUrlsProviderErrorHandler implements HttpResponseErrorHandler { 9 | @Override 10 | public boolean canHandle(@NotNull ResponseAdapter responseWrapper) { 11 | final String header = responseWrapper.getHeader(S3Constants.ERROR_SOURCE_HEADER_NAME); 12 | return header != null && (S3Constants.ErrorSource.TEAMCITY.name().equals(header) || S3Constants.ErrorSource.SDK.name().equals(header)); 13 | } 14 | 15 | @NotNull 16 | @Override 17 | public HttpClientUtil.HttpErrorCodeException handle(@NotNull ResponseAdapter responseWrapper) { 18 | final String response = responseWrapper.getResponse(); 19 | if (response != null) { 20 | return new HttpClientUtil.HttpErrorCodeException(responseWrapper.getStatusCode(), response, false, 21 | response.contains(AgentServerSharedErrorMessages.buildIsAlreadyFinishedOrDoesNotExist())); 22 | } else { 23 | //When TeamCity unloads the plugin, the endpoint for presigned urls unloads too, thus resulting in server responding 404 for url generation requests 24 | return new HttpClientUtil.HttpErrorCodeException(responseWrapper.getStatusCode(), null, OUR_RECOVERABLE_STATUS_CODES.contains(responseWrapper.getStatusCode()) 25 | || responseWrapper.getStatusCode() == 404); 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /s3-artifact-storage-agent/src/test/java/jetbrains/buildServer/artifacts/s3/publish/presigned/util/S3MultipartUploadFileSplitterTest.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish.presigned.util; 2 | 3 | import java.io.File; 4 | import java.io.FileOutputStream; 5 | import java.io.IOException; 6 | import java.nio.file.Files; 7 | import jetbrains.buildServer.BaseTestCase; 8 | import jetbrains.buildServer.agent.IOUtil; 9 | import org.testng.annotations.Test; 10 | 11 | @Test 12 | public class S3MultipartUploadFileSplitterTest extends BaseTestCase { 13 | public void splitFileWithDigests() throws IOException { 14 | final File file = new File(getClass().getClassLoader().getResource("artifacts/file.zip").getFile()); 15 | final File out = Files.createTempFile("tmp", "zip").toFile(); 16 | 17 | try (final FileOutputStream os = new FileOutputStream(out)) { 18 | final long length = file.length(); 19 | 20 | final S3MultipartUploadFileSplitter splitter = new S3MultipartUploadFileSplitter(length / 3); 21 | for (FilePart p : splitter.getFileParts(file, 3, true)) { 22 | p.write(os); 23 | } 24 | } 25 | 26 | assertEquals(IOUtil.readLines(file), IOUtil.readLines(out)); 27 | } 28 | 29 | 30 | public void splitFileWithoutDigests() throws IOException { 31 | final File file = new File(getClass().getClassLoader().getResource("artifacts/file.zip").getFile()); 32 | final File out = Files.createTempFile("tmp", "zip").toFile(); 33 | 34 | try (final FileOutputStream os = new FileOutputStream(out)) { 35 | final long length = file.length(); 36 | 37 | final S3MultipartUploadFileSplitter splitter = new S3MultipartUploadFileSplitter(length / 3); 38 | for (FilePart p : splitter.getFileParts(file, 3, false)) { 39 | p.write(os); 40 | } 41 | } 42 | 43 | assertEquals(IOUtil.readLines(file), IOUtil.readLines(out)); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/test/java/jetbrains/buildServer/artifacts/s3/S3DtoSerializerTest.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import jetbrains.buildServer.artifacts.s3.serialization.S3XmlSerializerFactory; 5 | import jetbrains.buildServer.artifacts.s3.serialization.XmlSerializer; 6 | import org.jdom.Element; 7 | import org.jdom.output.XMLOutputter; 8 | import org.testng.Assert; 9 | import org.testng.annotations.Test; 10 | 11 | @Test 12 | public class S3DtoSerializerTest { 13 | public void test_jdom_serialization() { 14 | final BucketLocationFetcher.BucketLocationDto actual = new BucketLocationFetcher.BucketLocationDto("bucketName", "bucketLocation"); 15 | 16 | XmlSerializer xmlSerializer = S3XmlSerializerFactory.getInstance(); 17 | final Element element = xmlSerializer.serializeAsElement(actual); 18 | final String jdom = new XMLOutputter().outputString(element); 19 | 20 | final BucketLocationFetcher.BucketLocationDto deserialized = xmlSerializer.deserialize(jdom, BucketLocationFetcher.BucketLocationDto.class); 21 | Assert.assertEquals(deserialized.getLocation(), actual.getLocation()); 22 | Assert.assertEquals(deserialized.getName(), actual.getName()); 23 | } 24 | 25 | public void test_jackson_serialization() throws JsonProcessingException { 26 | final BucketLocationFetcher.BucketLocationDto actual = new BucketLocationFetcher.BucketLocationDto("bucketName", "bucketLocation"); 27 | 28 | XmlSerializer xmlSerializer = S3XmlSerializerFactory.getInstance(); 29 | final String jackson = xmlSerializer.serialize(actual); 30 | 31 | final BucketLocationFetcher.BucketLocationDto deserialized = xmlSerializer.deserialize(jackson, BucketLocationFetcher.BucketLocationDto.class); 32 | Assert.assertEquals(deserialized.getLocation(), actual.getLocation()); 33 | Assert.assertEquals(deserialized.getName(), actual.getName()); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/App/components/StorageTypeChangedWarningDialog.tsx: -------------------------------------------------------------------------------- 1 | import { React } from '@jetbrains/teamcity-api'; 2 | import Dialog from '@jetbrains/ring-ui/components/dialog/dialog'; 3 | import { Content, Header } from '@jetbrains/ring-ui/components/island/island'; 4 | 5 | import Panel from '@jetbrains/ring-ui/components/panel/panel'; 6 | 7 | import Button from '@jetbrains/ring-ui/components/button/button'; 8 | 9 | import { AWS_S3, S3_COMPATIBLE } from '../Storage/components/StorageType'; 10 | import { useAppContext } from '../../contexts/AppContext'; 11 | import { FormFields } from '../appConstants'; 12 | import useS3Form from '../../hooks/useS3Form'; 13 | 14 | export default function StorageTypeChangedWarningDialog() { 15 | const [visited, setVisited] = React.useState(false); 16 | const config = useAppContext(); 17 | const { watch } = useS3Form(); 18 | const currentType = watch(FormFields.STORAGE_TYPE); 19 | const isS3Compatible = currentType?.key === S3_COMPATIBLE; 20 | const incorrectStorageTypeInProperties = React.useMemo( 21 | () => 22 | !config.isNewStorage && 23 | config.selectedStorageType === AWS_S3 && 24 | isS3Compatible, 25 | [config.isNewStorage, config.selectedStorageType, isS3Compatible] 26 | ); 27 | 28 | return ( 29 | setVisited(true)} 32 | autoFocusFirst 33 | dense 34 | trapFocus 35 | showCloseButton={false} 36 | > 37 |
{'Information'}
38 | 39 |
40 | { 41 | 'Your storage is not an S3 bucket hosted on Amazon Web Services. The storage type will be set to "Custom S3".' 42 | } 43 |
44 | 45 | 46 | 49 | 50 | 51 | ); 52 | } 53 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/cloudfront/CloudFrontEnabledPresignedUrlProvider.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.cloudfront; 2 | 3 | import java.io.IOException; 4 | import java.util.Map; 5 | 6 | import jetbrains.buildServer.artifacts.s3.PresignedUrlWithTtl; 7 | import org.jetbrains.annotations.NotNull; 8 | import org.jetbrains.annotations.Nullable; 9 | import software.amazon.awssdk.http.SdkHttpMethod; 10 | 11 | public interface CloudFrontEnabledPresignedUrlProvider { 12 | @NotNull 13 | PresignedUrlWithTtl generateDownloadUrl(@NotNull SdkHttpMethod httpMethod, 14 | @NotNull String objectKey, 15 | @NotNull CloudFrontSettings settings) throws IOException; 16 | 17 | @NotNull 18 | String generateUploadUrl(@NotNull String objectKey, @Nullable String digest, @NotNull CloudFrontSettings settings) throws IOException; 19 | 20 | @NotNull 21 | String generateUploadUrlForPart(@NotNull String objectKey, 22 | @Nullable String digest, 23 | int nPart, 24 | @NotNull String uploadId, 25 | @NotNull CloudFrontSettings settings) throws IOException; 26 | 27 | void finishMultipartUpload(@NotNull String uploadId, 28 | @NotNull String objectKey, 29 | @NotNull CloudFrontSettings settings, 30 | @Nullable String[] etags, 31 | boolean isSuccessful) throws IOException; 32 | 33 | @NotNull 34 | String startMultipartUpload(@NotNull String objectKey, @Nullable String contentType, @NotNull CloudFrontSettings settings) throws Exception; 35 | 36 | @NotNull 37 | CloudFrontSettings settings(@NotNull Map rawSettings, @NotNull Map projectSettings, @NotNull RequestMetadata metadata); 38 | } 39 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/publish/errors/S3ServerResponseErrorHandler.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish.errors; 2 | 3 | import jetbrains.buildServer.artifacts.s3.S3Constants; 4 | import jetbrains.buildServer.artifacts.s3.publish.presigned.util.HttpClientUtil; 5 | import jetbrains.buildServer.artifacts.s3.serialization.S3XmlSerializerFactory; 6 | import jetbrains.buildServer.artifacts.s3.transport.AmazonServiceErrorDto; 7 | import org.jetbrains.annotations.NotNull; 8 | import software.amazon.awssdk.awscore.exception.AwsServiceException; 9 | import software.amazon.awssdk.core.retry.RetryUtils; 10 | 11 | public class S3ServerResponseErrorHandler implements HttpResponseErrorHandler { 12 | @Override 13 | public boolean canHandle(@NotNull ResponseAdapter responseWrapper) { 14 | return S3Constants.ErrorSource.S3.name().equals(responseWrapper.getHeader(S3Constants.ERROR_SOURCE_HEADER_NAME)); 15 | } 16 | 17 | @NotNull 18 | @Override 19 | public HttpClientUtil.HttpErrorCodeException handle(@NotNull ResponseAdapter responseWrapper) { 20 | if (OUR_RECOVERABLE_STATUS_CODES.contains(responseWrapper.getStatusCode())) { 21 | return new HttpClientUtil.HttpErrorCodeException(responseWrapper.getStatusCode(), responseWrapper.getResponse(), true); 22 | } 23 | if (responseWrapper.getResponse() != null) { 24 | final AmazonServiceErrorDto deserialize = S3XmlSerializerFactory.getInstance().deserialize(responseWrapper.getResponse(), AmazonServiceErrorDto.class); 25 | final AwsServiceException exception = deserialize.toException(); 26 | final boolean isRecoverable = RetryUtils.isRetryableException(exception) || RetryUtils.isThrottlingException(exception); 27 | return new HttpClientUtil.HttpErrorCodeException(exception.statusCode(), exception.getMessage(), isRecoverable); 28 | } else { 29 | return new HttpClientUtil.HttpErrorCodeException(responseWrapper.getStatusCode(), null, false); 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/publish/errors/S3DirectResponseErrorHandler.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish.errors; 2 | 3 | import jetbrains.buildServer.artifacts.s3.publish.presigned.util.HttpClientUtil; 4 | import jetbrains.buildServer.artifacts.s3.publish.presigned.util.S3ErrorDto; 5 | import jetbrains.buildServer.artifacts.s3.serialization.S3XmlSerializerFactory; 6 | import org.jetbrains.annotations.NotNull; 7 | import software.amazon.awssdk.awscore.exception.AwsServiceException; 8 | import software.amazon.awssdk.core.retry.RetryUtils; 9 | 10 | /** 11 | * Error handler for handling S3 errors that are coming from S3 itself and not TeamCity server 12 | */ 13 | public class S3DirectResponseErrorHandler implements HttpResponseErrorHandler { 14 | @Override 15 | public boolean canHandle(@NotNull ResponseAdapter responseWrapper) { 16 | return responseWrapper.getHeader("x-amz-request-id") != null; 17 | } 18 | 19 | @NotNull 20 | @Override 21 | public HttpClientUtil.HttpErrorCodeException handle(@NotNull ResponseAdapter responseWrapper) { 22 | if (responseWrapper.getResponse() != null) { 23 | final S3ErrorDto deserialize = S3XmlSerializerFactory.getInstance().deserialize(responseWrapper.getResponse(), S3ErrorDto.class); 24 | deserialize.setCode(String.valueOf(responseWrapper.getStatusCode())); 25 | final AwsServiceException exception = deserialize.toException(); 26 | final boolean isRequestExpired = exception.statusCode() == 403 && "Request has expired".equals(exception.awsErrorDetails().errorMessage()); 27 | final boolean isRecoverable = isRequestExpired || RetryUtils.isRetryableException(exception) || RetryUtils.isThrottlingException(exception); 28 | return new HttpClientUtil.HttpErrorCodeException(exception.statusCode(), exception.getMessage(), isRecoverable); 29 | } else { 30 | return new HttpClientUtil.HttpErrorCodeException(responseWrapper.getStatusCode(), null, false); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/hooks/useBucketOptions.tsx: -------------------------------------------------------------------------------- 1 | import { React } from '@jetbrains/teamcity-api'; 2 | 3 | import { 4 | errorMessage, 5 | Option, 6 | } from '@jetbrains-internal/tcci-react-ui-components'; 7 | 8 | import { useFormContext } from 'react-hook-form'; 9 | 10 | import { ResponseErrors } from '@jetbrains-internal/tcci-react-ui-components/dist/types'; 11 | 12 | import { loadBucketList } from '../Utilities/fetchBucketNames'; 13 | import { IFormInput } from '../types'; 14 | 15 | import { useAppContext } from '../contexts/AppContext'; 16 | 17 | export default function useBucketOptions() { 18 | const config = useAppContext(); 19 | const { getValues } = useFormContext(); 20 | const [bucketsListLoading, setBucketsListLoading] = React.useState(false); 21 | const [buckets, setBuckets] = React.useState([]); 22 | const [loadErrors, setLoadErrors] = React.useState< 23 | ResponseErrors | string | undefined 24 | >(); 25 | 26 | const reloadBuckets = React.useCallback(async () => { 27 | if (bucketsListLoading) { 28 | return []; 29 | } 30 | 31 | setBucketsListLoading(true); 32 | setBuckets([]); 33 | setLoadErrors(undefined); 34 | try { 35 | const { bucketNames, errors } = await loadBucketList(config, getValues()); 36 | if (bucketNames) { 37 | const bucketsData = bucketNames.reduce((acc, cur) => { 38 | acc.push({ label: cur, key: cur }); 39 | return acc; 40 | }, [] as Option[]); 41 | setBuckets(bucketsData); 42 | return bucketsData; 43 | } 44 | if (errors) { 45 | setLoadErrors(errors); 46 | } 47 | } catch (e) { 48 | setLoadErrors(errorMessage(e)); 49 | } finally { 50 | setBucketsListLoading(false); 51 | } 52 | 53 | return []; 54 | }, [bucketsListLoading, config, getValues]); 55 | 56 | return { 57 | bucketOptions: buckets, 58 | loading: bucketsListLoading, 59 | errors: loadErrors, 60 | reload: reloadBuckets, 61 | }; 62 | } 63 | -------------------------------------------------------------------------------- /s3-artifact-storage-agent/src/main/java/jetbrains/buildServer/artifacts/s3/publish/SettingsProcessor.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | import jetbrains.buildServer.agent.BuildAgentConfiguration; 6 | import jetbrains.buildServer.agent.ServerProvidedProperties; 7 | import jetbrains.buildServer.agent.ssl.TrustedCertificatesDirectory; 8 | import jetbrains.buildServer.artifacts.s3.S3Configuration; 9 | import jetbrains.buildServer.artifacts.s3.SSLParamUtil; 10 | import jetbrains.buildServer.util.amazon.S3Util; 11 | import org.jetbrains.annotations.NotNull; 12 | 13 | import static jetbrains.buildServer.artifacts.s3.publish.S3FileUploader.configuration; 14 | 15 | public class SettingsProcessor { 16 | 17 | @NotNull private final BuildAgentConfiguration myBuildAgentConfiguration; 18 | 19 | public SettingsProcessor(@NotNull BuildAgentConfiguration buildAgentConfiguration) { 20 | myBuildAgentConfiguration = buildAgentConfiguration; 21 | } 22 | 23 | @NotNull 24 | public S3Configuration processSettings(@NotNull final Map sharedConfigParameters, @NotNull final Map artifactStorageSettings) { 25 | final String certDirectory = TrustedCertificatesDirectory.getAllCertificatesDirectory(myBuildAgentConfiguration); 26 | final Map storageSettings = new HashMap<>(SSLParamUtil.putSslDirectory(artifactStorageSettings, certDirectory)); 27 | final S3Util.S3AdvancedConfiguration s3AdvancedConfiguration = configuration(sharedConfigParameters, storageSettings); 28 | 29 | String projectId = sharedConfigParameters.get(ServerProvidedProperties.TEAMCITY_PROJECT_ID_PARAM); 30 | int nThreadsForFileParts = jetbrains.buildServer.artifacts.s3.S3Util.getNumberOfThreadsForFileParts(sharedConfigParameters); 31 | 32 | final S3Configuration s3Configuration = new S3Configuration(s3AdvancedConfiguration, storageSettings, projectId, nThreadsForFileParts); 33 | s3Configuration.validate(); 34 | return s3Configuration; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /s3-artifact-storage-agent/src/test/java/jetbrains/buildServer/artifacts/s3/download/S3MockContainer.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.download; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | import org.testcontainers.containers.GenericContainer; 5 | import org.testcontainers.containers.wait.strategy.Wait; 6 | import org.testcontainers.utility.DockerImageName; 7 | 8 | public final class S3MockContainer extends GenericContainer { 9 | private static final int S3MOCK_DEFAULT_HTTP_PORT = 9090; 10 | private static final int S3MOCK_DEFAULT_HTTPS_PORT = 9191; 11 | private static final DockerImageName DEFAULT_IMAGE_NAME = DockerImageName.parse("adobe/s3mock"); 12 | 13 | public S3MockContainer(@NotNull String tag) { 14 | this(DEFAULT_IMAGE_NAME.withTag(tag)); 15 | } 16 | 17 | public S3MockContainer(@NotNull DockerImageName dockerImageName) { 18 | super(dockerImageName); 19 | dockerImageName.assertCompatibleWith(DEFAULT_IMAGE_NAME); 20 | addExposedPort(S3MOCK_DEFAULT_HTTP_PORT); 21 | addExposedPort(S3MOCK_DEFAULT_HTTPS_PORT); 22 | waitingFor(Wait.forHttp("/favicon.ico").forPort(S3MOCK_DEFAULT_HTTP_PORT).withMethod("GET").forStatusCode(200)); 23 | } 24 | 25 | public S3MockContainer withRetainFilesOnExit(boolean retainFilesOnExit) { 26 | addEnv("retainFilesOnExit", String.valueOf(retainFilesOnExit)); 27 | return self(); 28 | } 29 | 30 | public S3MockContainer withInitialBuckets(String initialBuckets) { 31 | addEnv("initialBuckets", initialBuckets); 32 | return self(); 33 | } 34 | 35 | public String getHttpEndpoint() { 36 | return String.format("http://%s:%d", getHost(), getHttpServerPort()); 37 | } 38 | 39 | public String getHttpsEndpoint() { 40 | return String.format("https://%s:%d", getHost(), getHttpsServerPort()); 41 | } 42 | 43 | public Integer getHttpServerPort() { 44 | return getMappedPort(S3MOCK_DEFAULT_HTTP_PORT); 45 | } 46 | 47 | public Integer getHttpsServerPort() { 48 | return getMappedPort(S3MOCK_DEFAULT_HTTPS_PORT); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/publish/presigned/util/S3ErrorDto.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish.presigned.util; 2 | 3 | import software.amazon.awssdk.awscore.exception.AwsServiceException; 4 | import software.amazon.awssdk.awscore.exception.AwsErrorDetails; 5 | import javax.xml.bind.annotation.XmlElement; 6 | import javax.xml.bind.annotation.XmlRootElement; 7 | import org.jetbrains.annotations.NotNull; 8 | 9 | @XmlRootElement(name = "error") 10 | public class S3ErrorDto { 11 | @XmlElement(name = "Code") 12 | private String code; 13 | @XmlElement(name = "Message") 14 | private String message; 15 | @XmlElement(name = "RequestId") 16 | private String requestId; 17 | @XmlElement(name = "HostId") 18 | private String hostId; 19 | 20 | public String getCode() { 21 | return code; 22 | } 23 | 24 | public void setCode(String code) { 25 | this.code = code; 26 | } 27 | 28 | public String getMessage() { 29 | return message; 30 | } 31 | 32 | public void setMessage(String message) { 33 | this.message = message; 34 | } 35 | 36 | public String getRequestId() { 37 | return requestId; 38 | } 39 | 40 | public void setRequestId(String requestId) { 41 | this.requestId = requestId; 42 | } 43 | 44 | public String getHostId() { 45 | return hostId; 46 | } 47 | 48 | public void setHostId(String hostId) { 49 | this.hostId = hostId; 50 | } 51 | 52 | @NotNull 53 | public AwsServiceException toException() { 54 | AwsErrorDetails details = AwsErrorDetails.builder() 55 | .errorMessage(message) 56 | .errorCode(code) 57 | .build(); 58 | 59 | return AwsServiceException.builder() 60 | .awsErrorDetails(details) 61 | .requestId(requestId) 62 | .extendedRequestId(hostId) 63 | .build(); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/contexts/AppContext.tsx: -------------------------------------------------------------------------------- 1 | import { React } from '@jetbrains/teamcity-api'; 2 | import { ReactNode, useContext } from 'react'; 3 | 4 | import { Config } from '../types'; 5 | 6 | const AppContext = React.createContext({ 7 | storageTypes: '', 8 | storageNames: '', 9 | containersPath: '', 10 | distributionPath: '', 11 | publicKey: '', 12 | projectId: '', 13 | isNewStorage: false, 14 | cloudfrontFeatureOn: false, 15 | transferAccelerationOn: false, 16 | selectedStorageType: '', 17 | selectedStorageName: '', 18 | storageSettingsId: '', 19 | environmentNameValue: '', 20 | serviceEndpointValue: '', 21 | awsRegionName: '', 22 | showDefaultCredentialsChain: false, 23 | isDefaultCredentialsChain: false, 24 | credentialsTypeValue: '', 25 | accessKeyIdValue: '', 26 | secretAcessKeyValue: '', 27 | iamRoleArnValue: '', 28 | externalIdValue: '', 29 | bucketNameWasProvidedAsString: '', 30 | bucket: '', 31 | bucketPathPrefix: '', 32 | useCloudFront: false, 33 | cloudFrontUploadDistribution: '', 34 | cloudFrontDownloadDistribution: '', 35 | cloudFrontPublicKeyId: '', 36 | cloudFrontPrivateKey: '', 37 | usePresignUrlsForUpload: false, 38 | forceVirtualHostAddressing: false, 39 | enableAccelerateMode: false, 40 | multipartUploadThreshold: '', 41 | multipartUploadPartSize: '', 42 | chosenAwsConnectionId: '', 43 | availableAwsConnectionsControllerUrl: '', 44 | availableAwsConnectionsControllerResource: '', 45 | readOnly: false, 46 | verifyIntegrityAfterUpload: false, 47 | testConnectionUrl: '', 48 | postConnectionUrl: '', 49 | regionCodes: '', 50 | regionDescriptions: '', 51 | }); 52 | 53 | const { Provider } = AppContext; 54 | 55 | interface OwnProps { 56 | value: Config; 57 | children: ReactNode | ReactNode[]; 58 | } 59 | 60 | function AppContextProvider(props: OwnProps) { 61 | return {props.children}; 62 | } 63 | 64 | const useAppContext = () => useContext(AppContext); 65 | 66 | export { AppContextProvider, useAppContext }; 67 | -------------------------------------------------------------------------------- /modules/s3-artifact-storage-common.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /s3-artifact-storage-agent/src/main/java/jetbrains/buildServer/artifacts/s3/publish/S3CompatibleArtifactsPublisher.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.publish; 2 | 3 | import jetbrains.buildServer.ExtensionHolder; 4 | import jetbrains.buildServer.agent.AgentLifeCycleListener; 5 | import jetbrains.buildServer.agent.BuildAgentConfiguration; 6 | import jetbrains.buildServer.agent.CurrentBuildTracker; 7 | import jetbrains.buildServer.agent.artifacts.AgentArtifactHelper; 8 | import jetbrains.buildServer.artifacts.s3.lens.integration.LensIntegrationService; 9 | import jetbrains.buildServer.artifacts.s3.publish.presigned.upload.PresignedUrlsProviderClientFactory; 10 | import jetbrains.buildServer.util.EventDispatcher; 11 | import org.jetbrains.annotations.NotNull; 12 | 13 | import static jetbrains.buildServer.artifacts.s3.S3Constants.S3_COMPATIBLE_STORAGE_TYPE; 14 | 15 | /** 16 | * Duplicate S3ArtifactsPublisher to use S3 Compatible storage.type 17 | */ 18 | public class S3CompatibleArtifactsPublisher extends S3ArtifactsPublisher { 19 | public S3CompatibleArtifactsPublisher(@NotNull AgentArtifactHelper helper, 20 | @NotNull EventDispatcher dispatcher, 21 | @NotNull CurrentBuildTracker tracker, 22 | @NotNull BuildAgentConfiguration buildAgentConfiguration, 23 | @NotNull PresignedUrlsProviderClientFactory presignedUrlsProviderClient, 24 | @NotNull S3FileUploaderFactory uploaderFactory, 25 | @NotNull LensIntegrationService lensIntegrationService, 26 | @NotNull ExtensionHolder extensionHolder) { 27 | super(helper, dispatcher, tracker, buildAgentConfiguration, presignedUrlsProviderClient, uploaderFactory, lensIntegrationService, extensionHolder); 28 | } 29 | 30 | @NotNull 31 | @Override 32 | public String getType() { 33 | return S3_COMPATIBLE_STORAGE_TYPE; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /modules/lens-integration.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/transport/PresignedUrlListResponseDto.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.transport; 2 | 3 | import java.util.Collection; 4 | import java.util.List; 5 | import javax.xml.bind.annotation.XmlAccessType; 6 | import javax.xml.bind.annotation.XmlAccessorType; 7 | import javax.xml.bind.annotation.XmlRootElement; 8 | import jetbrains.buildServer.Used; 9 | import org.jetbrains.annotations.NotNull; 10 | 11 | @XmlRootElement(name = "presignedUrlListResponse") 12 | @XmlAccessorType(XmlAccessType.PROPERTY) 13 | public class PresignedUrlListResponseDto { 14 | private Collection presignedUrls; 15 | private boolean isVersion2; 16 | 17 | @Used("serialization") 18 | public PresignedUrlListResponseDto() { 19 | } 20 | 21 | private PresignedUrlListResponseDto(@NotNull final Collection presignedUrls, final boolean isVersion2) { 22 | this.presignedUrls = presignedUrls; 23 | this.isVersion2 = isVersion2; 24 | } 25 | 26 | @NotNull 27 | public static PresignedUrlListResponseDto createV1(@NotNull final List presignedUrls) { 28 | if (presignedUrls.stream().anyMatch(presignedUrlResponseDto -> presignedUrlResponseDto.getUploadId() != null)) { 29 | throw new IllegalArgumentException("Multipart upload is only supported in version2 of presigned url api"); 30 | } 31 | return new PresignedUrlListResponseDto(presignedUrls, false); 32 | } 33 | 34 | @NotNull 35 | public static PresignedUrlListResponseDto createV2(@NotNull final List presignedUrls) { 36 | return new PresignedUrlListResponseDto(presignedUrls, true); 37 | } 38 | 39 | public boolean isVersion2() { 40 | return isVersion2; 41 | } 42 | 43 | public void setVersion2(boolean version2) { 44 | isVersion2 = version2; 45 | } 46 | 47 | @NotNull 48 | public Collection getPresignedUrls() { 49 | return presignedUrls; 50 | } 51 | 52 | public void setPresignedUrls(@NotNull Collection presignedUrls) { 53 | this.presignedUrls = presignedUrls; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /s3-artifact-storage-server/src/main/java/jetbrains/buildServer/artifacts/s3/settings/S3PropertiesProcessor.java: -------------------------------------------------------------------------------- 1 | package jetbrains.buildServer.artifacts.s3.settings; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Collection; 5 | import java.util.HashMap; 6 | import java.util.Map; 7 | import jetbrains.buildServer.artifacts.s3.S3Util; 8 | import jetbrains.buildServer.artifacts.s3.cloudfront.CloudFrontConstants; 9 | import jetbrains.buildServer.clouds.amazon.connector.featureDevelopment.ChosenAwsConnPropertiesProcessor; 10 | import jetbrains.buildServer.clouds.amazon.connector.utils.parameters.ParamUtil; 11 | import jetbrains.buildServer.serverSide.InvalidProperty; 12 | import jetbrains.buildServer.serverSide.PropertiesProcessor; 13 | import jetbrains.buildServer.util.StringUtil; 14 | import jetbrains.buildServer.util.amazon.AWSCommonParams; 15 | 16 | public class S3PropertiesProcessor implements PropertiesProcessor { 17 | 18 | @Override 19 | public Collection process(Map params) { 20 | final ArrayList invalids = new ArrayList<>(); 21 | 22 | if (ParamUtil.withAwsConnectionId(params)) { 23 | invalids.addAll(new ChosenAwsConnPropertiesProcessor().process(params)); 24 | } else { 25 | final Map awsErrors = new HashMap<>(); 26 | if (!StringUtil.isTrue(params.get(AWSCommonParams.USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN_PARAM))){ 27 | AWSCommonParams.verifyAccessKeys(params, awsErrors); 28 | } 29 | for (Map.Entry e : awsErrors.entrySet()) { 30 | invalids.add(new InvalidProperty(e.getKey(), e.getValue())); 31 | } 32 | } 33 | 34 | for (Map.Entry e : S3Util.validateParameters(params, true).entrySet()) { 35 | invalids.add(new InvalidProperty(e.getKey(), e.getValue())); 36 | } 37 | 38 | final String bucketName = S3Util.getBucketName(params); 39 | if (bucketName != null) { 40 | if (CloudFrontConstants.isEnabled() && S3Util.getCloudFrontEnabled(params)) { 41 | invalids.addAll(new CloudFrontPropertiesProcessor().process(params)); 42 | } 43 | } 44 | 45 | return invalids; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /s3-artifact-storage-server/kotlin-dsl/S3CommonSettings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Storage name 10 | 11 | 12 | 13 | 14 | Bucket name 15 | 16 | 17 | 18 | 19 | Bucket path prefix 20 | 21 | 22 | 23 | 24 | Whether to use Pre-Signed URLs to upload 25 | 26 | 27 | 28 | 29 | Whether to force Virtual Host Addressing 30 | 31 | 32 | 33 | 34 | Whether to verify integrity of artifacts after upload 35 | 36 | 37 | 38 | 39 | Initiates multipart upload for files larger than the specified value. 40 | Minimum value is 5MB. Allowed suffixes: KB, MB, GB, TB. 41 | Leave empty to use the default value. 42 | 43 | 44 | 45 | 46 | Specify the maximum allowed part size. Minimum value is 5MB. 47 | Allowed suffixes: KB, MB, GB, TB. Leave empty to use the default value. 48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /s3-artifact-storage-ui/src/App/Storage/components/StorageType.tsx: -------------------------------------------------------------------------------- 1 | import { React } from '@jetbrains/teamcity-api'; 2 | import { SelectItem } from '@jetbrains/ring-ui/components/select/select'; 3 | import { useFormContext } from 'react-hook-form'; 4 | 5 | import { 6 | FormRow, 7 | FormSelect, 8 | Option, 9 | } from '@jetbrains-internal/tcci-react-ui-components'; 10 | 11 | import { useMemo, useCallback } from 'react'; 12 | 13 | import useStorageOptions from '../../../hooks/useStorageOptions'; 14 | import { IFormInput } from '../../../types'; 15 | import { AWS_ENV_TYPE_ARRAY, FormFields } from '../../appConstants'; 16 | 17 | type StorageTypeConfig = { 18 | onChange: (option: Option | null) => void | undefined | null; 19 | }; 20 | 21 | export const S3_COMPATIBLE = 'S3_storage_compatible'; 22 | export const AWS_S3 = 'S3_storage'; 23 | 24 | export default function StorageType({ onChange: callback }: StorageTypeConfig) { 25 | const storageOptions = useStorageOptions(); 26 | const s3StorageTypes = useMemo(() => [AWS_S3, S3_COMPATIBLE], []); 27 | 28 | const { control, setValue, clearErrors } = useFormContext(); 29 | 30 | const innerOnChange = useCallback( 31 | (option: SelectItem