├── .github ├── PULL_REQUEST_TEMPLATE └── workflows │ ├── build.yml │ ├── parallel.yml │ └── sequential.yml ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── bin ├── get-metrics.py ├── restart-coordinator.sh ├── restart-shuffle-server.sh ├── rss-env.sh ├── start-coordinator.sh ├── start-shuffle-server.sh ├── stop-coordinator.sh ├── stop-shuffle-server.sh └── utils.sh ├── build_distribution.sh ├── checkstyle-suppressions.xml ├── checkstyle.xml ├── client-mr ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── hadoop │ │ ├── mapred │ │ ├── RssMapOutputCollector.java │ │ ├── SortWriteBuffer.java │ │ └── SortWriteBufferManager.java │ │ └── mapreduce │ │ ├── MRIdHelper.java │ │ ├── RssMRConfig.java │ │ ├── RssMRUtils.java │ │ ├── task │ │ └── reduce │ │ │ ├── RssBypassWriter.java │ │ │ ├── RssEventFetcher.java │ │ │ ├── RssFetcher.java │ │ │ └── RssShuffle.java │ │ └── v2 │ │ └── app │ │ └── RssMRAppMaster.java │ └── test │ └── java │ └── org │ └── apache │ └── hadoop │ ├── mapred │ ├── SortWriteBufferManagerTest.java │ └── SortWriteBufferTest.java │ └── mapreduce │ ├── RssMRUtilsTest.java │ └── task │ └── reduce │ ├── EvenFetcherTest.java │ └── FetcherTest.java ├── client-spark ├── common │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ └── shuffle │ │ │ ├── RssShuffleHandle.java │ │ │ ├── RssSparkConfig.java │ │ │ ├── RssSparkShuffleUtils.java │ │ │ ├── reader │ │ │ └── RssShuffleDataIterator.java │ │ │ └── writer │ │ │ ├── AddBlockEvent.java │ │ │ ├── BufferManagerOptions.java │ │ │ ├── WrappedByteArrayOutputStream.java │ │ │ ├── WriteBufferManager.java │ │ │ └── WriterBuffer.java │ │ └── test │ │ └── java │ │ └── org │ │ └── apache │ │ └── spark │ │ └── shuffle │ │ ├── RssSparkShuffleUtilsTest.java │ │ ├── reader │ │ ├── AbstractRssReaderTest.java │ │ └── RssShuffleDataIteratorTest.java │ │ └── writer │ │ ├── WriteBufferManagerTest.java │ │ └── WriteBufferTest.java ├── spark2 │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ └── shuffle │ │ │ ├── DelegationRssShuffleManager.java │ │ │ ├── RssShuffleManager.java │ │ │ ├── reader │ │ │ └── RssShuffleReader.java │ │ │ └── writer │ │ │ └── RssShuffleWriter.java │ │ └── test │ │ └── java │ │ └── org │ │ └── apache │ │ └── spark │ │ └── shuffle │ │ ├── DelegationRssShuffleManagerTest.java │ │ ├── reader │ │ └── RssShuffleReaderTest.java │ │ └── writer │ │ └── RssShuffleWriterTest.java └── spark3 │ ├── pom.xml │ └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── spark │ │ └── shuffle │ │ ├── DelegationRssShuffleManager.java │ │ ├── RssShuffleManager.java │ │ ├── reader │ │ └── RssShuffleReader.java │ │ └── writer │ │ └── RssShuffleWriter.java │ └── test │ └── java │ └── org │ └── apache │ └── spark │ └── shuffle │ ├── DelegationRssShuffleManagerTest.java │ ├── TestUtils.java │ ├── reader │ └── RssShuffleReaderTest.java │ └── writer │ └── RssShuffleWriterTest.java ├── client ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── tencent │ │ └── rss │ │ └── client │ │ ├── api │ │ ├── ShuffleReadClient.java │ │ └── ShuffleWriteClient.java │ │ ├── factory │ │ └── ShuffleClientFactory.java │ │ ├── impl │ │ ├── ShuffleReadClientImpl.java │ │ └── ShuffleWriteClientImpl.java │ │ ├── request │ │ └── CreateShuffleReadClientRequest.java │ │ ├── response │ │ ├── CompressedShuffleBlock.java │ │ └── SendShuffleDataResult.java │ │ └── util │ │ ├── ClientUtils.java │ │ ├── DefaultIdHelper.java │ │ ├── IdHelper.java │ │ └── RssClientConfig.java │ └── test │ └── java │ └── com │ └── tencent │ └── rss │ └── client │ ├── ClientUtilsTest.java │ ├── TestUtils.java │ └── impl │ ├── ShuffleReadClientImplTest.java │ └── ShuffleWriteClientImplTest.java ├── common ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── tencent │ │ └── rss │ │ └── common │ │ ├── Arguments.java │ │ ├── BufferSegment.java │ │ ├── PartitionRange.java │ │ ├── RemoteStorageInfo.java │ │ ├── RssShuffleUtils.java │ │ ├── ShuffleAssignmentsInfo.java │ │ ├── ShuffleBlockInfo.java │ │ ├── ShuffleDataResult.java │ │ ├── ShuffleDataSegment.java │ │ ├── ShuffleIndexResult.java │ │ ├── ShufflePartitionedBlock.java │ │ ├── ShufflePartitionedData.java │ │ ├── ShuffleRegisterInfo.java │ │ ├── ShuffleServerInfo.java │ │ ├── config │ │ ├── ConfigOption.java │ │ ├── ConfigOptions.java │ │ ├── ConfigUtils.java │ │ ├── RssBaseConf.java │ │ └── RssConf.java │ │ ├── exception │ │ └── RssException.java │ │ ├── metrics │ │ ├── GRPCMetrics.java │ │ ├── JvmMetrics.java │ │ └── MetricsManager.java │ │ ├── rpc │ │ ├── GrpcServer.java │ │ ├── MonitoringServerCall.java │ │ ├── MonitoringServerCallListener.java │ │ ├── MonitoringServerInterceptor.java │ │ └── ServerInterface.java │ │ ├── util │ │ ├── ByteUnit.java │ │ ├── ChecksumUtils.java │ │ ├── Constants.java │ │ ├── ExitUtils.java │ │ ├── RssUtils.java │ │ └── UnitConverter.java │ │ └── web │ │ ├── CommonMetricsServlet.java │ │ └── JettyServer.java │ └── test │ ├── java │ └── com │ │ └── tencent │ │ └── rss │ │ └── common │ │ ├── ArgumentsTest.java │ │ ├── RemoteStorageInfoTest.java │ │ ├── RssShuffleUtilsTest.java │ │ ├── ShufflePartitionedBlockTest.java │ │ ├── config │ │ ├── ConfigOptionTest.java │ │ └── RssConfTest.java │ │ ├── metrics │ │ ├── MetricsManagerTest.java │ │ └── TestUtils.java │ │ ├── util │ │ ├── ChecksumUtilsTest.java │ │ ├── ExitUtilsTest.java │ │ ├── RssUtilsTest.java │ │ └── UnitConverterTest.java │ │ └── web │ │ └── JettyServerTest.java │ └── resources │ ├── rss-defaults.conf │ └── server.conf ├── conf ├── coordinator.conf ├── log4j.properties └── server.conf ├── coordinator ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── tencent │ │ └── rss │ │ └── coordinator │ │ ├── AccessCandidatesChecker.java │ │ ├── AccessCheckResult.java │ │ ├── AccessChecker.java │ │ ├── AccessClusterLoadChecker.java │ │ ├── AccessInfo.java │ │ ├── AccessManager.java │ │ ├── ApplicationManager.java │ │ ├── AssignmentStrategy.java │ │ ├── AssignmentStrategyFactory.java │ │ ├── BasicAssignmentStrategy.java │ │ ├── ClientConfManager.java │ │ ├── ClusterManager.java │ │ ├── ClusterManagerFactory.java │ │ ├── CoordinatorConf.java │ │ ├── CoordinatorFactory.java │ │ ├── CoordinatorGrpcMetrics.java │ │ ├── CoordinatorGrpcService.java │ │ ├── CoordinatorMetrics.java │ │ ├── CoordinatorServer.java │ │ ├── CoordinatorUtils.java │ │ ├── PartitionBalanceAssignmentStrategy.java │ │ ├── PartitionRangeAssignment.java │ │ ├── ServerNode.java │ │ └── SimpleClusterManager.java │ └── test │ ├── java │ └── com │ │ └── tencent │ │ └── rss │ │ └── coordinator │ │ ├── AccessCandidatesCheckerTest.java │ │ ├── AccessClusterLoadCheckerTest.java │ │ ├── AccessManagerTest.java │ │ ├── ApplicationManagerTest.java │ │ ├── BasicAssignmentStrategyTest.java │ │ ├── ClientConfManagerTest.java │ │ ├── CoordinatorConfTest.java │ │ ├── CoordinatorMetricsTest.java │ │ ├── CoordinatorServerTest.java │ │ ├── CoordinatorUtilsTest.java │ │ ├── PartitionBalanceAssignmentStrategyTest.java │ │ ├── PartitionRangeAssignmentTest.java │ │ ├── PartitionRangeTest.java │ │ ├── ServerNodeTest.java │ │ └── SimpleClusterManagerTest.java │ └── resources │ ├── coordinator.conf │ └── empty ├── docs ├── _config.yml ├── asset │ ├── rss_architecture.png │ ├── rss_data_format.png │ ├── rss_shuffle_write.png │ └── wechat.png ├── client_guide.md ├── coordinator_guide.md ├── gen-doc.sh ├── index.md └── server_guide.md ├── integration-test ├── common │ ├── pom.xml │ └── src │ │ └── test │ │ └── java │ │ └── com │ │ └── tencent │ │ └── rss │ │ └── test │ │ ├── AccessCandidatesCheckerHdfsTest.java │ │ ├── AccessClusterTest.java │ │ ├── ClientConfManagerHdfsTest.java │ │ ├── CoordinatorGrpcTest.java │ │ ├── CoordinatorTestBase.java │ │ ├── CoordinatorTestUtils.java │ │ ├── DiskErrorToleranceTest.java │ │ ├── FetchClientConfTest.java │ │ ├── HealthCheckCoordinatorGrpcTest.java │ │ ├── IntegrationTestBase.java │ │ ├── MultiStorageFaultToleranceTest.java │ │ ├── MultiStorageTest.java │ │ ├── PartitionBalanceCoordinatorGrpcTest.java │ │ ├── QuorumTest.java │ │ ├── ShuffleReadWriteBase.java │ │ ├── ShuffleServerGrpcTest.java │ │ ├── ShuffleServerWithHdfsTest.java │ │ ├── ShuffleServerWithLocalTest.java │ │ ├── ShuffleServerWithMemLocalHdfsTest.java │ │ ├── ShuffleServerWithMemoryTest.java │ │ └── ShuffleWithRssClientTest.java ├── mr │ ├── pom.xml │ └── src │ │ └── test │ │ └── java │ │ └── com │ │ └── tencent │ │ └── rss │ │ └── test │ │ ├── FailoverAppMaster.java │ │ ├── LargeSorterTest.java │ │ ├── MRIntegrationTestBase.java │ │ ├── SecondarySortTest.java │ │ └── WordCountTest.java ├── spark-common │ ├── pom.xml │ └── src │ │ └── test │ │ ├── java │ │ └── com │ │ │ └── tencent │ │ │ └── rss │ │ │ └── test │ │ │ ├── AutoAccessTest.java │ │ │ ├── CombineByKeyTest.java │ │ │ ├── DynamicFetchClientConfTest.java │ │ │ ├── GroupByKeyTest.java │ │ │ ├── RepartitionTest.java │ │ │ ├── RepartitionWithHdfsMultiStorageRssTest.java │ │ │ ├── RepartitionWithLocalFileRssTest.java │ │ │ ├── RepartitionWithMemoryMultiStorageRssTest.java │ │ │ ├── RepartitionWithMemoryRssTest.java │ │ │ ├── RepartitionWithUploadMultiStorageRssTest.java │ │ │ ├── SimpleTestBase.java │ │ │ ├── SparkClientWithLocalTest.java │ │ │ ├── SparkFallbackReadTest.java │ │ │ ├── SparkIntegrationTestBase.java │ │ │ ├── SparkSQLMultiStorageRssTest.java │ │ │ ├── SparkSQLTest.java │ │ │ ├── SparkSQLWithDelegationShuffleManager.java │ │ │ ├── SparkSQLWithDelegationShuffleManagerFallback.java │ │ │ ├── SparkSQLWithMemoryLocalTest.java │ │ │ └── TestUtils.java │ │ └── resources │ │ └── candidates ├── spark2 │ ├── pom.xml │ └── src │ │ └── test │ │ └── java │ │ └── com │ │ └── tencent │ │ └── rss │ │ └── test │ │ └── GetReaderTest.java └── spark3 │ ├── pom.xml │ └── src │ └── test │ └── java │ └── com │ └── tencent │ └── rss │ └── test │ ├── AQERepartitionTest.java │ ├── AQESkewedJoinTest.java │ └── GetReaderTest.java ├── internal-client ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── tencent │ └── rss │ └── client │ ├── api │ ├── CoordinatorClient.java │ └── ShuffleServerClient.java │ ├── factory │ ├── CoordinatorClientFactory.java │ └── ShuffleServerClientFactory.java │ ├── impl │ └── grpc │ │ ├── CoordinatorGrpcClient.java │ │ ├── GrpcClient.java │ │ └── ShuffleServerGrpcClient.java │ ├── request │ ├── RssAccessClusterRequest.java │ ├── RssAppHeartBeatRequest.java │ ├── RssFetchClientConfRequest.java │ ├── RssFetchRemoteStorageRequest.java │ ├── RssFinishShuffleRequest.java │ ├── RssGetInMemoryShuffleDataRequest.java │ ├── RssGetShuffleAssignmentsRequest.java │ ├── RssGetShuffleDataRequest.java │ ├── RssGetShuffleIndexRequest.java │ ├── RssGetShuffleResultRequest.java │ ├── RssRegisterShuffleRequest.java │ ├── RssReportShuffleResultRequest.java │ ├── RssSendCommitRequest.java │ ├── RssSendHeartBeatRequest.java │ └── RssSendShuffleDataRequest.java │ ├── response │ ├── ClientResponse.java │ ├── ResponseStatusCode.java │ ├── RssAccessClusterResponse.java │ ├── RssAppHeartBeatResponse.java │ ├── RssFetchClientConfResponse.java │ ├── RssFetchRemoteStorageResponse.java │ ├── RssFinishShuffleResponse.java │ ├── RssGetInMemoryShuffleDataResponse.java │ ├── RssGetShuffleAssignmentsResponse.java │ ├── RssGetShuffleDataResponse.java │ ├── RssGetShuffleIndexResponse.java │ ├── RssGetShuffleResultResponse.java │ ├── RssRegisterShuffleResponse.java │ ├── RssReportShuffleResultResponse.java │ ├── RssSendCommitResponse.java │ ├── RssSendHeartBeatResponse.java │ └── RssSendShuffleDataResponse.java │ └── util │ └── ClientType.java ├── pom.xml ├── proto ├── pom.xml └── src │ └── main │ └── proto │ └── Rss.proto ├── server ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── tencent │ │ └── rss │ │ └── server │ │ ├── Checker.java │ │ ├── HealthCheck.java │ │ ├── LocalStorageChecker.java │ │ ├── RegisterHeartBeat.java │ │ ├── ShuffleDataFlushEvent.java │ │ ├── ShuffleDataReadEvent.java │ │ ├── ShuffleFlushManager.java │ │ ├── ShuffleServer.java │ │ ├── ShuffleServerConf.java │ │ ├── ShuffleServerFactory.java │ │ ├── ShuffleServerGrpcMetrics.java │ │ ├── ShuffleServerGrpcService.java │ │ ├── ShuffleServerMetrics.java │ │ ├── ShuffleTaskManager.java │ │ ├── ShuffleUploader.java │ │ ├── StatusCode.java │ │ ├── buffer │ │ ├── PreAllocatedBufferInfo.java │ │ ├── ShuffleBuffer.java │ │ └── ShuffleBufferManager.java │ │ └── storage │ │ ├── HdfsStorageManager.java │ │ ├── LocalStorageManager.java │ │ ├── MultiStorageManager.java │ │ ├── SingleStorageManager.java │ │ ├── StorageManager.java │ │ └── StorageManagerFactory.java │ └── test │ ├── java │ └── com │ │ └── tencent │ │ └── rss │ │ └── server │ │ ├── HealthCheckTest.java │ │ ├── HealthyMockChecker.java │ │ ├── MockedGrpcServer.java │ │ ├── MockedShuffleServer.java │ │ ├── MockedShuffleServerFactory.java │ │ ├── MockedShuffleServerGrpcService.java │ │ ├── ShuffleFlushManagerTest.java │ │ ├── ShuffleServerConfTest.java │ │ ├── ShuffleServerMetricsTest.java │ │ ├── ShuffleServerTest.java │ │ ├── ShuffleTaskManagerTest.java │ │ ├── ShuffleUploaderTest.java │ │ ├── StorageCheckerTest.java │ │ ├── UnHealthyMockChecker.java │ │ ├── buffer │ │ ├── BufferTestBase.java │ │ ├── ShuffleBufferManagerTest.java │ │ └── ShuffleBufferTest.java │ │ └── storage │ │ ├── HdfsStorageManagerTest.java │ │ └── MultiStorageManagerTest.java │ └── resources │ ├── confBySizeStringTest.conf │ ├── confTest.conf │ ├── gcTest.conf │ └── server.conf ├── spark-patches ├── spark-2.4.6_dynamic_allocation_support.patch ├── spark-3.1.2_dynamic_allocation_support.patch └── spark-3.2.1_dynamic_allocation_support.patch ├── spotbugs-exclude.xml └── storage ├── pom.xml └── src ├── main └── java │ └── com │ └── tencent │ └── rss │ └── storage │ ├── api │ └── FileReader.java │ ├── common │ ├── AbstractStorage.java │ ├── FileBasedShuffleSegment.java │ ├── HdfsStorage.java │ ├── LocalStorage.java │ ├── LocalStorageMeta.java │ ├── ShuffleFileInfo.java │ ├── ShuffleInfo.java │ ├── ShuffleSegment.java │ ├── Storage.java │ ├── StorageReadMetrics.java │ └── StorageWriteMetrics.java │ ├── factory │ ├── ShuffleHandlerFactory.java │ └── ShuffleUploadHandlerFactory.java │ ├── handler │ ├── api │ │ ├── ClientReadHandler.java │ │ ├── ServerReadHandler.java │ │ ├── ShuffleDeleteHandler.java │ │ ├── ShuffleUploadHandler.java │ │ └── ShuffleWriteHandler.java │ └── impl │ │ ├── AbstractClientReadHandler.java │ │ ├── ComposedClientReadHandler.java │ │ ├── DataFileSegment.java │ │ ├── DataSkippableReadHandler.java │ │ ├── FileSegment.java │ │ ├── HdfsClientReadHandler.java │ │ ├── HdfsFileReader.java │ │ ├── HdfsFileWriter.java │ │ ├── HdfsShuffleDeleteHandler.java │ │ ├── HdfsShuffleReadHandler.java │ │ ├── HdfsShuffleUploadHandler.java │ │ ├── HdfsShuffleWriteHandler.java │ │ ├── LocalFileClientReadHandler.java │ │ ├── LocalFileDeleteHandler.java │ │ ├── LocalFileQuorumClientReadHandler.java │ │ ├── LocalFileReader.java │ │ ├── LocalFileServerReadHandler.java │ │ ├── LocalFileWriteHandler.java │ │ ├── LocalFileWriter.java │ │ ├── MemoryClientReadHandler.java │ │ ├── MemoryQuorumClientReadHandler.java │ │ ├── ShuffleIndexHeader.java │ │ ├── UploadedHdfsClientReadHandler.java │ │ └── UploadedStorageHdfsShuffleReadHandler.java │ ├── request │ ├── CreateShuffleDeleteHandlerRequest.java │ ├── CreateShuffleReadHandlerRequest.java │ ├── CreateShuffleUploadHandlerRequest.java │ └── CreateShuffleWriteHandlerRequest.java │ └── util │ ├── ShuffleStorageUtils.java │ ├── ShuffleUploadResult.java │ └── StorageType.java └── test └── java └── com └── tencent └── rss └── storage ├── HdfsShuffleHandlerTestBase.java ├── HdfsTestBase.java ├── common ├── LocalStorageTest.java └── ShuffleFileInfoTest.java ├── handler └── impl │ ├── HdfsClientReadHandlerTest.java │ ├── HdfsFileReaderTest.java │ ├── HdfsFileWriterTest.java │ ├── HdfsHandlerTest.java │ ├── HdfsShuffleReadHandlerTest.java │ ├── HdfsShuffleUploadHandlerTest.java │ ├── LocalFileHandlerTest.java │ ├── MultiStorageHdfsClientReadHandlerTest.java │ └── MultiStorageHdfsShuffleReadHandlerTest.java └── util ├── ShuffleHdfsStorageUtilsTest.java └── ShuffleStorageUtilsTest.java /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Tencent is pleased to support the open source community by making 3 | # Firestorm-Spark remote shuffle server available. 4 | # 5 | # Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | # this file except in compliance with the License. You may obtain a copy of the 9 | # License at 10 | # 11 | # https://opensource.org/licenses/Apache-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | # WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations under the License. 17 | # 18 | # This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time 19 | # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven 20 | 21 | name: CI 22 | 23 | on: [push, pull_request] 24 | 25 | jobs: 26 | checkstyle: 27 | uses: ./.github/workflows/sequential.yml 28 | with: 29 | maven-args: checkstyle:check 30 | 31 | license: 32 | uses: ./.github/workflows/sequential.yml 33 | with: 34 | maven-args: org.apache.rat:apache-rat-plugin:check 35 | summary: "grep -r '!?????' --include='rat.txt' | awk '{print $3}'" 36 | 37 | spotbugs: 38 | uses: ./.github/workflows/sequential.yml 39 | with: 40 | maven-args: test-compile spotbugs:check 41 | experimental: true 42 | 43 | build: 44 | uses: ./.github/workflows/parallel.yml 45 | with: 46 | maven-args: package 47 | reports-path: "**/target/surefire-reports/*.txt" 48 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.ipr 2 | *.iml 3 | *.iws 4 | *.pyc 5 | *.pyo 6 | .idea/ 7 | .idea_modules/ 8 | .settings 9 | .cache 10 | .DS_Store 11 | .project 12 | .classpath 13 | .scala_dependencies 14 | docs/api/ 15 | lib_managed/ 16 | logs/ 17 | src_managed/ 18 | target/ 19 | reports/ 20 | metastore_db/ 21 | derby.log 22 | dependency-reduced-pom.xml 23 | rss-*.tgz 24 | 25 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Firestorm 2 | Welcome to [report Issues](https://github.com/Tencent/Firestorm/issues) or [pull requests](https://github.com/Tencent/Firestorm/pulls). It's recommended to read the following Contributing Guide first before contributing. 3 | 4 | ## Issues 5 | We use Github Issues to track public bugs and feature requests. 6 | 7 | ### Search Known Issues First 8 | Please search on [report Issues](https://github.com/Tencent/Firestorm/issues) to avoid creating duplicate issue. 9 | 10 | ### Reporting New Issues 11 | * Be sure to include a title and clear description, as much relevant information as possible. 12 | * A code sample or an executable test case demonstrating is high recommended. 13 | 14 | ## Pull Requests 15 | * We use master branch as our developing branch. 16 | * Do not commit/push directly to the master branch. Instead, create a fork and file a pull request. 17 | * When maintaining a branch, merge frequently with the master. 18 | * When maintaining a branch, submit pull requests to the master frequently. 19 | * If you are working on a bigger issue try to split it up into several smaller issues. 20 | * We reserve full and final discretion over whether or not we will merge a pull request. Adhering to these guidelines is not a complete guarantee that your pull request will be merged. 21 | 22 | ### Make Pull Requests 23 | The code team will monitor all pull request, we run some code check and test on it. After all tests passed, we will accecpt this PR. But it won't merge to `master` branch at once, which have some delay. 24 | 25 | Before submitting a pull request, please make sure the followings are done: 26 | 27 | 1. Fork the repo and create your branch from `master`. 28 | 2. Update code or documentation if you have changed APIs. 29 | 3. Add the copyright notice to the top of any new files you've added. 30 | 4. Check your code lints and checkstyles. 31 | 5. Test and test again your code. 32 | 33 | ## Code Style Guide 34 | Use [Code Style](https://github.com/Tencent/Firestorm/blob/master/checkstyle.xml) for Java. 35 | 36 | * 2 spaces for indentation rather than tabs 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Firestorm is moved to Apache Uniffle 2 | 3 | Firestorm is now moved to ASF and renamed to [Apache Uniffle (incubating)](https://github.com/apache/incubator-uniffle). 4 | For the latest content, please visit Apache Uniffle mailing lists, site and repository: 5 | 6 | - Mailing list: dev@uniffle.apache.org ([subscribe][1] / [unsubscribe][2] / [archives][3]) 7 | - Site: https://uniffle.apache.org 8 | - Repository: https://github.com/apache/incubator-uniffle 9 | 10 | [1]: mailto:dev-subscribe@uniffle.apache.org?subject=(send%20this%20email%20to%20subscribe) 11 | [2]: mailto:dev-unsubscribe@uniffle.apache.org?subject=(send%20this%20email%20to%20unsubscribe) 12 | [3]: https://lists.apache.org/list.html?dev@uniffle.apache.org 13 | -------------------------------------------------------------------------------- /bin/restart-coordinator.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # Tencent is pleased to support the open source community by making 5 | # Firestorm-Spark remote shuffle server available. 6 | # 7 | # Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use 10 | # this file except in compliance with the License. You may obtain a copy of the 11 | # License at 12 | # 13 | # https://opensource.org/licenses/Apache-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 17 | # WARRANTIES OF ANY KIND, either express or implied. See the License for the 18 | # specific language governing permissions and limitations under the License. 19 | # 20 | 21 | set -o pipefail 22 | set -e 23 | 24 | COORDINATOR_HOME="$( 25 | cd "$(dirname "$0")/.." 26 | pwd 27 | )" 28 | 29 | cd $COORDINATOR_HOME 30 | 31 | bash ./bin/stop-coordinator.sh 32 | sleep 3 33 | bash ./bin/start-coordinator.sh 34 | 35 | exit 0 -------------------------------------------------------------------------------- /bin/restart-shuffle-server.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # Tencent is pleased to support the open source community by making 5 | # Firestorm-Spark remote shuffle server available. 6 | # 7 | # Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use 10 | # this file except in compliance with the License. You may obtain a copy of the 11 | # License at 12 | # 13 | # https://opensource.org/licenses/Apache-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 17 | # WARRANTIES OF ANY KIND, either express or implied. See the License for the 18 | # specific language governing permissions and limitations under the License. 19 | # 20 | 21 | set -o pipefail 22 | set -e 23 | 24 | SHUFFLE_SERVER_HOME="$( 25 | cd "$(dirname "$0")/.." 26 | pwd 27 | )" 28 | 29 | cd $SHUFFLE_SERVER_HOME 30 | 31 | bash ./bin/stop-shuffle-server.sh 32 | sleep 3 33 | bash ./bin/start-shuffle-server.sh 34 | 35 | exit 0 -------------------------------------------------------------------------------- /bin/rss-env.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # Tencent is pleased to support the open source community by making 5 | # Firestorm-Spark remote shuffle server available. 6 | # 7 | # Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use 10 | # this file except in compliance with the License. You may obtain a copy of the 11 | # License at 12 | # 13 | # https://opensource.org/licenses/Apache-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 17 | # WARRANTIES OF ANY KIND, either express or implied. See the License for the 18 | # specific language governing permissions and limitations under the License. 19 | # 20 | 21 | set -o pipefail 22 | set -e 23 | 24 | JAVA_HOME= 25 | HADOOP_HOME= 26 | XMX_SIZE="80g" 27 | 28 | RUNNER="${JAVA_HOME}/bin/java" 29 | JPS="${JAVA_HOME}/bin/jps" 30 | -------------------------------------------------------------------------------- /bin/stop-coordinator.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # Tencent is pleased to support the open source community by making 5 | # Firestorm-Spark remote shuffle server available. 6 | # 7 | # Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use 10 | # this file except in compliance with the License. You may obtain a copy of the 11 | # License at 12 | # 13 | # https://opensource.org/licenses/Apache-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 17 | # WARRANTIES OF ANY KIND, either express or implied. See the License for the 18 | # specific language governing permissions and limitations under the License. 19 | # 20 | 21 | set -o pipefail 22 | set -o nounset # exit the script if you try to use an uninitialised variable 23 | set -o errexit # exit the script if any statement returns a non-true return value 24 | set -e 25 | 26 | SCRIPT_DIR="$(cd "`dirname "$0"`"; pwd)" 27 | BASE_DIR="$(cd "`dirname "$0"`/.."; pwd)" 28 | 29 | source "$SCRIPT_DIR/utils.sh" 30 | common_shutdown "coordinator" ${BASE_DIR} -------------------------------------------------------------------------------- /bin/stop-shuffle-server.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # Tencent is pleased to support the open source community by making 5 | # Firestorm-Spark remote shuffle server available. 6 | # 7 | # Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use 10 | # this file except in compliance with the License. You may obtain a copy of the 11 | # License at 12 | # 13 | # https://opensource.org/licenses/Apache-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 17 | # WARRANTIES OF ANY KIND, either express or implied. See the License for the 18 | # specific language governing permissions and limitations under the License. 19 | # 20 | 21 | set -o pipefail 22 | set -o nounset # exit the script if you try to use an uninitialised variable 23 | set -o errexit # exit the script if any statement returns a non-true return value 24 | set -e 25 | 26 | SCRIPT_DIR="$(cd "`dirname "$0"`"; pwd)" 27 | BASE_DIR="$(cd "`dirname "$0"`/.."; pwd)" 28 | 29 | source "$SCRIPT_DIR/utils.sh" 30 | common_shutdown "shuffle-server" ${BASE_DIR} -------------------------------------------------------------------------------- /checkstyle-suppressions.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /client-mr/src/main/java/org/apache/hadoop/mapreduce/MRIdHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package org.apache.hadoop.mapreduce; 20 | 21 | import com.tencent.rss.client.util.IdHelper; 22 | 23 | public class MRIdHelper implements IdHelper { 24 | 25 | @Override 26 | public long getTaskAttemptId(long blockId) { 27 | return RssMRUtils.getTaskAttemptId(blockId); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /client-spark/common/src/main/java/org/apache/spark/shuffle/writer/AddBlockEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package org.apache.spark.shuffle.writer; 20 | 21 | import java.util.List; 22 | 23 | import com.tencent.rss.common.ShuffleBlockInfo; 24 | 25 | public class AddBlockEvent { 26 | 27 | private String taskId; 28 | private List shuffleDataInfoList; 29 | 30 | public AddBlockEvent(String taskId, List shuffleDataInfoList) { 31 | this.taskId = taskId; 32 | this.shuffleDataInfoList = shuffleDataInfoList; 33 | } 34 | 35 | public String getTaskId() { 36 | return taskId; 37 | } 38 | 39 | public List getShuffleDataInfoList() { 40 | return shuffleDataInfoList; 41 | } 42 | 43 | @Override 44 | public String toString() { 45 | return "AddBlockEvent: TaskId[" + taskId + "], " + shuffleDataInfoList; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /client-spark/common/src/main/java/org/apache/spark/shuffle/writer/WrappedByteArrayOutputStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package org.apache.spark.shuffle.writer; 20 | 21 | import java.io.ByteArrayOutputStream; 22 | 23 | /** 24 | * Subclass of ByteArrayOutputStream that exposes `buf` directly. 25 | */ 26 | public class WrappedByteArrayOutputStream extends ByteArrayOutputStream { 27 | 28 | public WrappedByteArrayOutputStream(int size) { 29 | super(size); 30 | } 31 | 32 | public byte[] getBuf() { 33 | return buf; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /client-spark/spark3/src/test/java/org/apache/spark/shuffle/TestUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package org.apache.spark.shuffle; 20 | 21 | import java.util.Map; 22 | import java.util.Set; 23 | 24 | import org.apache.spark.SparkConf; 25 | import org.apache.spark.util.EventLoop; 26 | 27 | public class TestUtils { 28 | 29 | private TestUtils() { 30 | } 31 | 32 | public static RssShuffleManager createShuffleManager( 33 | SparkConf conf, 34 | Boolean isDriver, 35 | EventLoop loop, 36 | Map> successBlockIds, 37 | Map> failBlockIds) { 38 | return new RssShuffleManager(conf, isDriver, loop, successBlockIds, failBlockIds); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /client/src/main/java/com/tencent/rss/client/api/ShuffleReadClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.api; 20 | 21 | import com.tencent.rss.client.response.CompressedShuffleBlock; 22 | 23 | public interface ShuffleReadClient { 24 | 25 | CompressedShuffleBlock readShuffleBlockData(); 26 | 27 | void checkProcessedBlockIds(); 28 | 29 | void close(); 30 | 31 | void logStatics(); 32 | } 33 | -------------------------------------------------------------------------------- /client/src/main/java/com/tencent/rss/client/response/CompressedShuffleBlock.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.response; 20 | 21 | import java.nio.ByteBuffer; 22 | 23 | public class CompressedShuffleBlock { 24 | 25 | private ByteBuffer byteBuffer; 26 | private int uncompressLength; 27 | 28 | public CompressedShuffleBlock(ByteBuffer byteBuffer, int uncompressLength) { 29 | this.byteBuffer = byteBuffer; 30 | this.uncompressLength = uncompressLength; 31 | } 32 | 33 | public int getUncompressLength() { 34 | return uncompressLength; 35 | } 36 | 37 | public ByteBuffer getByteBuffer() { 38 | return byteBuffer; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /client/src/main/java/com/tencent/rss/client/response/SendShuffleDataResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.response; 20 | 21 | import java.util.Set; 22 | 23 | public class SendShuffleDataResult { 24 | 25 | private Set successBlockIds; 26 | private Set failedBlockIds; 27 | 28 | public SendShuffleDataResult(Set successBlockIds, Set failedBlockIds) { 29 | this.successBlockIds = successBlockIds; 30 | this.failedBlockIds = failedBlockIds; 31 | } 32 | 33 | public Set getSuccessBlockIds() { 34 | return successBlockIds; 35 | } 36 | 37 | public Set getFailedBlockIds() { 38 | return failedBlockIds; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /client/src/main/java/com/tencent/rss/client/util/DefaultIdHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.util; 20 | 21 | import com.tencent.rss.common.util.Constants; 22 | 23 | public class DefaultIdHelper implements IdHelper { 24 | @Override 25 | public long getTaskAttemptId(long blockId) { 26 | return blockId & Constants.MAX_TASK_ATTEMPT_ID; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /client/src/main/java/com/tencent/rss/client/util/IdHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.util; 20 | 21 | public interface IdHelper { 22 | 23 | long getTaskAttemptId(long blockId); 24 | } 25 | -------------------------------------------------------------------------------- /common/src/main/java/com/tencent/rss/common/Arguments.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.common; 20 | 21 | import picocli.CommandLine.Option; 22 | 23 | public class Arguments { 24 | 25 | @Option(names = {"-c", "--conf"}, description = "config file") 26 | private String configFile; 27 | 28 | public String getConfigFile() { 29 | return this.configFile; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /common/src/main/java/com/tencent/rss/common/PartitionRange.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.common; 20 | 21 | import java.util.Objects; 22 | 23 | /** 24 | * Class for partition range: [start, end] 25 | * Note: both inclusive 26 | */ 27 | public class PartitionRange implements Comparable { 28 | 29 | private final int start; 30 | private final int end; 31 | 32 | public PartitionRange(int start, int end) { 33 | if (start > end || start < 0) { 34 | throw new IllegalArgumentException("Illegal partition range [start, end]"); 35 | } 36 | this.start = start; 37 | this.end = end; 38 | } 39 | 40 | public int getStart() { 41 | return start; 42 | } 43 | 44 | public int getEnd() { 45 | return end; 46 | } 47 | 48 | public int getPartitionNum() { 49 | return end - start + 1; 50 | } 51 | 52 | @Override 53 | public boolean equals(Object o) { 54 | if (this == o) { 55 | return true; 56 | } 57 | if (o == null || getClass() != o.getClass()) { 58 | return false; 59 | } 60 | PartitionRange that = (PartitionRange) o; 61 | return start == that.start && end == that.end; 62 | } 63 | 64 | @Override 65 | public int hashCode() { 66 | return Objects.hash(start, end); 67 | } 68 | 69 | @Override 70 | public String toString() { 71 | return "PartitionRange[" + start + ", " + end + ']'; 72 | } 73 | 74 | @Override 75 | public int compareTo(PartitionRange o) { 76 | return this.getStart() - o.getStart(); 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /common/src/main/java/com/tencent/rss/common/ShuffleAssignmentsInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.common; 20 | 21 | import java.util.List; 22 | import java.util.Map; 23 | 24 | public class ShuffleAssignmentsInfo { 25 | 26 | private Map> partitionToServers; 27 | private Map> serverToPartitionRanges; 28 | 29 | public ShuffleAssignmentsInfo( 30 | Map> partitionToServers, 31 | Map> serverToPartitionRanges) { 32 | this.partitionToServers = partitionToServers; 33 | this.serverToPartitionRanges = serverToPartitionRanges; 34 | } 35 | 36 | public Map> getPartitionToServers() { 37 | return partitionToServers; 38 | } 39 | 40 | public Map> getServerToPartitionRanges() { 41 | return serverToPartitionRanges; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /common/src/main/java/com/tencent/rss/common/ShuffleDataResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.common; 20 | 21 | import java.util.List; 22 | 23 | import com.google.common.collect.Lists; 24 | 25 | public class ShuffleDataResult { 26 | 27 | private final byte[] data; 28 | private final List bufferSegments; 29 | 30 | public ShuffleDataResult() { 31 | this(new byte[0]); 32 | } 33 | 34 | public ShuffleDataResult(byte[] data) { 35 | this(data, Lists.newArrayList()); 36 | } 37 | 38 | public ShuffleDataResult(byte[] data, List bufferSegments) { 39 | this.data = data; 40 | this.bufferSegments = bufferSegments; 41 | } 42 | 43 | public byte[] getData() { 44 | return data; 45 | } 46 | 47 | public List getBufferSegments() { 48 | return bufferSegments; 49 | } 50 | 51 | public boolean isEmpty() { 52 | return bufferSegments == null || bufferSegments.isEmpty() || data == null || data.length == 0; 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /common/src/main/java/com/tencent/rss/common/ShuffleDataSegment.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.common; 20 | 21 | import java.util.List; 22 | 23 | /** 24 | * ShuffleDataSegment is a view of a segment of shuffle data file, which is split according to the read buffer size. 25 | * It contains a list of BufferSegment, they are indices of the block in the data file segment. 26 | */ 27 | public class ShuffleDataSegment { 28 | private final long offset; 29 | private final int length; 30 | private final List bufferSegments; 31 | 32 | public ShuffleDataSegment(long offset, int length, List bufferSegments) { 33 | this.offset = offset; 34 | this.length = length; 35 | this.bufferSegments = bufferSegments; 36 | } 37 | 38 | public long getOffset() { 39 | return offset; 40 | } 41 | 42 | public int getLength() { 43 | return length; 44 | } 45 | 46 | public List getBufferSegments() { 47 | return bufferSegments; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /common/src/main/java/com/tencent/rss/common/ShuffleIndexResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.common; 20 | 21 | public class ShuffleIndexResult { 22 | private final byte[] indexData; 23 | 24 | public ShuffleIndexResult() { 25 | this(new byte[0]); 26 | } 27 | 28 | public ShuffleIndexResult(byte[] bytes) { 29 | this.indexData = bytes; 30 | } 31 | 32 | public byte[] getIndexData() { 33 | return indexData; 34 | } 35 | 36 | public boolean isEmpty() { 37 | return indexData == null || indexData.length == 0; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /common/src/main/java/com/tencent/rss/common/ShufflePartitionedData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.common; 20 | 21 | public class ShufflePartitionedData { 22 | 23 | private int partitionId; 24 | private ShufflePartitionedBlock[] blockList; 25 | 26 | public ShufflePartitionedData(int partitionId, ShufflePartitionedBlock[] blockList) { 27 | this.partitionId = partitionId; 28 | this.blockList = blockList; 29 | } 30 | 31 | @Override 32 | public String toString() { 33 | return "ShufflePartitionedData{partitionId=" + partitionId + ", blockList=" + blockList + '}'; 34 | } 35 | 36 | public int getPartitionId() { 37 | return partitionId; 38 | } 39 | 40 | public void setPartitionId(int partitionId) { 41 | this.partitionId = partitionId; 42 | } 43 | 44 | public ShufflePartitionedBlock[] getBlockList() { 45 | if (blockList == null) { 46 | return new ShufflePartitionedBlock[]{}; 47 | } 48 | return blockList; 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /common/src/main/java/com/tencent/rss/common/ShuffleRegisterInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.common; 20 | 21 | import java.util.List; 22 | 23 | public class ShuffleRegisterInfo { 24 | 25 | private ShuffleServerInfo shuffleServerInfo; 26 | private List partitionRanges; 27 | 28 | public ShuffleRegisterInfo(ShuffleServerInfo shuffleServerInfo, List partitionRanges) { 29 | this.shuffleServerInfo = shuffleServerInfo; 30 | this.partitionRanges = partitionRanges; 31 | } 32 | 33 | public ShuffleServerInfo getShuffleServerInfo() { 34 | return shuffleServerInfo; 35 | } 36 | 37 | public List getPartitionRanges() { 38 | return partitionRanges; 39 | } 40 | 41 | @Override 42 | public int hashCode() { 43 | return shuffleServerInfo.hashCode(); 44 | } 45 | 46 | @Override 47 | public boolean equals(Object obj) { 48 | if (obj instanceof ShuffleRegisterInfo) { 49 | return shuffleServerInfo.equals(((ShuffleRegisterInfo) obj).getShuffleServerInfo()) 50 | && partitionRanges.equals(((ShuffleRegisterInfo) obj).getPartitionRanges()); 51 | } 52 | return false; 53 | } 54 | 55 | @Override 56 | public String toString() { 57 | return "ShuffleRegisterInfo: shuffleServerInfo[" + shuffleServerInfo.getId() + "], " + partitionRanges; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /common/src/main/java/com/tencent/rss/common/ShuffleServerInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.common; 20 | 21 | import java.io.Serializable; 22 | 23 | public class ShuffleServerInfo implements Serializable { 24 | 25 | private String id; 26 | 27 | private String host; 28 | 29 | private int port; 30 | 31 | public ShuffleServerInfo(String id, String host, int port) { 32 | this.id = id; 33 | this.host = host; 34 | this.port = port; 35 | } 36 | 37 | public String getId() { 38 | return id; 39 | } 40 | 41 | public String getHost() { 42 | return host; 43 | } 44 | 45 | public int getPort() { 46 | return port; 47 | } 48 | 49 | @Override 50 | public int hashCode() { 51 | return host.hashCode(); 52 | } 53 | 54 | @Override 55 | public boolean equals(Object obj) { 56 | if (obj instanceof ShuffleServerInfo) { 57 | return id.equals(((ShuffleServerInfo) obj).getId()) 58 | && host.equals(((ShuffleServerInfo) obj).getHost()) 59 | && port == ((ShuffleServerInfo) obj).getPort(); 60 | } 61 | return false; 62 | } 63 | 64 | @Override 65 | public String toString() { 66 | return "ShuffleServerInfo{id[" + id + "], host[" + host + "], port[" + port + "]}"; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /common/src/main/java/com/tencent/rss/common/exception/RssException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | 20 | package com.tencent.rss.common.exception; 21 | 22 | public class RssException extends RuntimeException { 23 | 24 | public RssException(String message) { 25 | super(message); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /common/src/main/java/com/tencent/rss/common/rpc/MonitoringServerCall.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.common.rpc; 20 | 21 | import io.grpc.ForwardingServerCall; 22 | import io.grpc.Metadata; 23 | import io.grpc.ServerCall; 24 | import io.grpc.Status; 25 | 26 | import com.tencent.rss.common.metrics.GRPCMetrics; 27 | 28 | public class MonitoringServerCall extends ForwardingServerCall.SimpleForwardingServerCall { 29 | 30 | private final String methodName; 31 | private final GRPCMetrics grpcMetrics; 32 | 33 | protected MonitoringServerCall( 34 | ServerCall delegate, String methodName, GRPCMetrics grpcMetrics) { 35 | super(delegate); 36 | this.methodName = methodName; 37 | this.grpcMetrics = grpcMetrics; 38 | } 39 | 40 | @Override 41 | public void close(Status status, Metadata responseHeaders) { 42 | try { 43 | super.close(status, responseHeaders); 44 | } finally { 45 | grpcMetrics.decCounter(methodName); 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /common/src/main/java/com/tencent/rss/common/rpc/MonitoringServerCallListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.common.rpc; 20 | 21 | import io.grpc.ForwardingServerCallListener; 22 | import io.grpc.ServerCall; 23 | 24 | public class MonitoringServerCallListener extends ForwardingServerCallListener { 25 | 26 | private final ServerCall.Listener delegate; 27 | 28 | public MonitoringServerCallListener(ServerCall.Listener delegate) { 29 | this.delegate = delegate; 30 | } 31 | 32 | @Override 33 | protected ServerCall.Listener delegate() { 34 | return delegate; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /common/src/main/java/com/tencent/rss/common/rpc/MonitoringServerInterceptor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.common.rpc; 20 | 21 | import io.grpc.Metadata; 22 | import io.grpc.MethodDescriptor; 23 | import io.grpc.ServerCall; 24 | import io.grpc.ServerCallHandler; 25 | import io.grpc.ServerInterceptor; 26 | 27 | import com.tencent.rss.common.metrics.GRPCMetrics; 28 | 29 | public class MonitoringServerInterceptor implements ServerInterceptor { 30 | 31 | private final GRPCMetrics grpcMetrics; 32 | 33 | public MonitoringServerInterceptor(GRPCMetrics grpcMetrics) { 34 | this.grpcMetrics = grpcMetrics; 35 | } 36 | 37 | @Override 38 | public ServerCall.Listener interceptCall( 39 | ServerCall serverCall, 40 | Metadata metadata, 41 | ServerCallHandler serverCallHandler) { 42 | MethodDescriptor methodDescriptor = serverCall.getMethodDescriptor(); 43 | String methodName = methodDescriptor.getBareMethodName(); 44 | // a call is coming 45 | grpcMetrics.incCounter(methodName); 46 | MonitoringServerCall monitoringServerCall = 47 | new MonitoringServerCall<>(serverCall, methodName, grpcMetrics); 48 | return new MonitoringServerCallListener<>( 49 | serverCallHandler.startCall(monitoringServerCall, metadata)); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /common/src/main/java/com/tencent/rss/common/rpc/ServerInterface.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.common.rpc; 20 | 21 | import java.io.IOException; 22 | 23 | public interface ServerInterface { 24 | 25 | void start() throws IOException; 26 | 27 | void stop() throws InterruptedException; 28 | 29 | void blockUntilShutdown() throws InterruptedException; 30 | } 31 | -------------------------------------------------------------------------------- /common/src/main/java/com/tencent/rss/common/util/ChecksumUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.common.util; 20 | 21 | import java.nio.ByteBuffer; 22 | import java.util.zip.CRC32; 23 | 24 | public class ChecksumUtils { 25 | 26 | private static final int LENGTH_PER_CRC = 4 * 1024; 27 | 28 | public static long getCrc32(byte[] buf) { 29 | return getCrc32(buf, 0, buf.length); 30 | } 31 | 32 | public static long getCrc32(byte[] buf, int offset, int length) { 33 | CRC32 crc32 = new CRC32(); 34 | 35 | for (int i = 0; i < length; ) { 36 | int len = Math.min(LENGTH_PER_CRC, length - i); 37 | crc32.update(buf, i + offset, len); 38 | i += len; 39 | } 40 | 41 | return crc32.getValue(); 42 | } 43 | 44 | // you may need to flip at first 45 | public static long getCrc32(ByteBuffer byteBuffer) { 46 | if (byteBuffer.hasArray()) { 47 | return getCrc32(byteBuffer.array(), byteBuffer.arrayOffset() + byteBuffer.position(), byteBuffer.remaining()); 48 | } else { 49 | byte[] byteArray = new byte[byteBuffer.remaining()]; 50 | byteBuffer.get(byteArray); 51 | return getCrc32(byteArray); 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /common/src/main/java/com/tencent/rss/common/util/ExitUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.common.util; 20 | 21 | import org.slf4j.Logger; 22 | 23 | public class ExitUtils { 24 | 25 | private static boolean isSystemExitDisabled = false; 26 | 27 | public static class ExitException extends RuntimeException { 28 | 29 | private final int status; 30 | 31 | ExitException(int status, String message, Throwable throwable) { 32 | super(message, throwable); 33 | this.status = status; 34 | } 35 | 36 | public int getStatus() { 37 | return status; 38 | } 39 | } 40 | 41 | /** 42 | * 43 | * @param status exit status 44 | * @param message terminate message 45 | * @param throwable throwable caused terminate 46 | * @param logger logger of the caller 47 | */ 48 | public static void terminate(int status, String message, Throwable throwable, Logger logger) throws ExitException { 49 | if (logger != null) { 50 | final String s = "Terminating with exit status " + status + ": " + message; 51 | if (status == 0) { 52 | logger.info(s, throwable); 53 | } else { 54 | logger.error(s, throwable); 55 | } 56 | } 57 | 58 | if (!isSystemExitDisabled) { 59 | System.exit(status); 60 | } 61 | 62 | throw new ExitException(status, message, throwable); 63 | } 64 | 65 | public static void disableSystemExit() { 66 | isSystemExitDisabled = true; 67 | } 68 | 69 | } 70 | -------------------------------------------------------------------------------- /common/src/test/java/com/tencent/rss/common/ArgumentsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.common; 20 | 21 | import static org.junit.jupiter.api.Assertions.assertEquals; 22 | import static org.junit.jupiter.api.Assertions.assertNull; 23 | 24 | import org.junit.jupiter.api.Test; 25 | import picocli.CommandLine; 26 | 27 | public class ArgumentsTest { 28 | 29 | private static final String confFile = ClassLoader.getSystemResource("server.conf").getFile(); 30 | 31 | @Test 32 | public void argTest() { 33 | String[] args = {"-c", confFile}; 34 | Arguments arguments = new Arguments(); 35 | CommandLine commandLine = new CommandLine(arguments); 36 | commandLine.parseArgs(args); 37 | assertEquals(confFile, arguments.getConfigFile()); 38 | } 39 | 40 | @Test 41 | public void argEmptyTest() { 42 | String[] args = new String[0]; 43 | Arguments arguments = new Arguments(); 44 | CommandLine commandLine = new CommandLine(arguments); 45 | commandLine.parseArgs(args); 46 | assertNull(arguments.getConfigFile()); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /common/src/test/java/com/tencent/rss/common/RssShuffleUtilsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.common; 20 | 21 | import com.google.common.collect.Lists; 22 | import org.junit.jupiter.api.Test; 23 | 24 | import java.util.List; 25 | import java.util.Random; 26 | 27 | import static org.junit.jupiter.api.Assertions.assertArrayEquals; 28 | 29 | public class RssShuffleUtilsTest { 30 | @Test 31 | public void compressionTest() { 32 | List testSizes = Lists.newArrayList( 33 | 1, 1024, 128 * 1024, 512 * 1024, 1024 * 1024, 4 * 1024 * 1024); 34 | for (int size : testSizes) { 35 | singleTest(size); 36 | } 37 | } 38 | 39 | private void singleTest(int size) { 40 | byte[] buf = new byte[size]; 41 | new Random().nextBytes(buf); 42 | 43 | byte[] compressed = RssShuffleUtils.compressData(buf); 44 | byte[] uncompressed = RssShuffleUtils.decompressData(compressed, size); 45 | assertArrayEquals(buf, uncompressed); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /common/src/test/java/com/tencent/rss/common/ShufflePartitionedBlockTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.common; 20 | 21 | import static org.junit.jupiter.api.Assertions.assertArrayEquals; 22 | import static org.junit.jupiter.api.Assertions.assertEquals; 23 | 24 | import java.util.Random; 25 | import org.junit.jupiter.api.Test; 26 | 27 | public class ShufflePartitionedBlockTest { 28 | 29 | @Test 30 | public void shufflePartitionedBlockTest() { 31 | byte[] buf = new byte[3]; 32 | new Random().nextBytes(buf); 33 | 34 | ShufflePartitionedBlock b1 = new ShufflePartitionedBlock(1, 1, 2, 3, 1, buf); 35 | assertEquals(1, b1.getLength()); 36 | assertEquals(2, b1.getCrc()); 37 | assertEquals(3, b1.getBlockId()); 38 | 39 | ShufflePartitionedBlock b3 = new ShufflePartitionedBlock(1, 1, 2, 3, 3, buf); 40 | assertArrayEquals(buf, b3.getData()); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /common/src/test/java/com/tencent/rss/common/metrics/TestUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.common.metrics; 20 | 21 | import java.io.BufferedReader; 22 | import java.io.IOException; 23 | import java.io.InputStreamReader; 24 | import java.net.HttpURLConnection; 25 | import java.net.URL; 26 | 27 | public class TestUtils { 28 | 29 | private TestUtils() { 30 | } 31 | 32 | public static String httpGetMetrics(String urlString) throws IOException { 33 | URL url = new URL(urlString); 34 | HttpURLConnection con = (HttpURLConnection) url.openConnection(); 35 | con.setRequestMethod("GET"); 36 | BufferedReader in = new BufferedReader( 37 | new InputStreamReader(con.getInputStream())); 38 | String inputLine; 39 | StringBuffer content = new StringBuffer(); 40 | while ((inputLine = in.readLine()) != null) { 41 | content.append(inputLine); 42 | } 43 | in.close(); 44 | return content.toString(); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /common/src/test/java/com/tencent/rss/common/util/ExitUtilsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.common.util; 20 | 21 | import static org.junit.jupiter.api.Assertions.assertEquals; 22 | import static org.junit.jupiter.api.Assertions.fail; 23 | 24 | import com.tencent.rss.common.util.ExitUtils.ExitException; 25 | 26 | import org.junit.jupiter.api.Test; 27 | 28 | public class ExitUtilsTest { 29 | 30 | @Test 31 | public void test() { 32 | try { 33 | final int status = -1; 34 | final String testExitMessage = "testExitMessage"; 35 | try { 36 | ExitUtils.disableSystemExit(); 37 | ExitUtils.terminate(status, testExitMessage, null, null); 38 | fail(); 39 | } catch (ExitException e) { 40 | assertEquals(status, e.getStatus()); 41 | assertEquals(testExitMessage, e.getMessage()); 42 | } 43 | 44 | final Thread t = new Thread(null, () -> { 45 | throw new AssertionError("TestUncaughtException"); 46 | }, "testThread"); 47 | t.start(); 48 | t.join(); 49 | } catch (Exception e) { 50 | e.printStackTrace(); 51 | fail(); 52 | } 53 | 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /common/src/test/resources/rss-defaults.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Tencent is pleased to support the open source community by making 3 | # Firestorm-Spark remote shuffle server available. 4 | # 5 | # Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | # this file except in compliance with the License. You may obtain a copy of the 9 | # License at 10 | # 11 | # https://opensource.org/licenses/Apache-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | # WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations under the License. 17 | # 18 | 19 | # Default system properties included when running rss. 20 | # This is useful for setting default environmental settings. 21 | 22 | # Example: 23 | rss.coordinator.port 12121 24 | rss.server.heartbeat.interval 155 25 | rss.x.y.z true 26 | rss.a.b.c.extraJavaOptions -XX:+PrintGCDetails-Dkey=value-Dnumbers="one two three" 27 | -------------------------------------------------------------------------------- /common/src/test/resources/server.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Tencent is pleased to support the open source community by making 3 | # Firestorm-Spark remote shuffle server available. 4 | # 5 | # Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | # this file except in compliance with the License. You may obtain a copy of the 9 | # License at 10 | # 11 | # https://opensource.org/licenses/Apache-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | # WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations under the License. 17 | # 18 | 19 | rss.jetty.http.port 9527 -------------------------------------------------------------------------------- /conf/coordinator.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Tencent is pleased to support the open source community by making 3 | # Firestorm-Spark remote shuffle server available. 4 | # 5 | # Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | # this file except in compliance with the License. You may obtain a copy of the 9 | # License at 10 | # 11 | # https://opensource.org/licenses/Apache-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | # WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations under the License. 17 | # 18 | 19 | rss.rpc.server.port 19999 20 | rss.jetty.http.port 19998 21 | rss.coordinator.server.heartbeat.timeout 30000 22 | rss.coordinator.app.expired 60000 23 | rss.coordinator.shuffle.nodes.max 13 24 | rss.coordinator.exclude.nodes.file.path file:///xxx 25 | -------------------------------------------------------------------------------- /conf/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Tencent is pleased to support the open source community by making 3 | # Firestorm-Spark remote shuffle server available. 4 | # 5 | # Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | # this file except in compliance with the License. You may obtain a copy of the 9 | # License at 10 | # 11 | # https://opensource.org/licenses/Apache-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | # WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations under the License. 17 | # 18 | 19 | log4j.rootCategory=INFO, RollingAppender 20 | log4j.appender.console=org.apache.log4j.ConsoleAppender 21 | log4j.appender.console.Threshold=INFO 22 | log4j.appender.console.target=System.err 23 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 24 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n 25 | log4j.appender.RollingAppender=org.apache.log4j.RollingFileAppender 26 | log4j.appender.RollingAppender.File=./logs/rss.log 27 | log4j.appender.RollingAppender.MaxFileSize=50MB 28 | log4j.appender.RollingAppender.MaxBackupIndex=10 29 | log4j.appender.RollingAppender.layout=org.apache.log4j.PatternLayout 30 | log4j.appender.RollingAppender.layout.ConversionPattern=[%p] %d %t %c{1} %M - %m%n 31 | -------------------------------------------------------------------------------- /conf/server.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Tencent is pleased to support the open source community by making 3 | # Firestorm-Spark remote shuffle server available. 4 | # 5 | # Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | # this file except in compliance with the License. You may obtain a copy of the 9 | # License at 10 | # 11 | # https://opensource.org/licenses/Apache-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | # WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations under the License. 17 | # 18 | 19 | rss.rpc.server.port 19999 20 | rss.jetty.http.port 19998 21 | rss.storage.basePath /xxx,/xxx 22 | rss.storage.type MEMORY_LOCALFILE_HDFS 23 | rss.coordinator.quorum xxx:19999,xxx:19999 24 | rss.server.buffer.capacity 40gb 25 | rss.server.read.buffer.capacity 20gb 26 | rss.server.flush.thread.alive 5 27 | rss.server.flush.threadPool.size 10 28 | rss.server.disk.capacity 1t 29 | -------------------------------------------------------------------------------- /coordinator/src/main/java/com/tencent/rss/coordinator/AccessCheckResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.coordinator; 20 | 21 | public class AccessCheckResult { 22 | 23 | private final boolean success; 24 | private final String msg; 25 | 26 | public AccessCheckResult(boolean success, String msg) { 27 | this.success = success; 28 | this.msg = msg; 29 | } 30 | 31 | public boolean isSuccess() { 32 | return success; 33 | } 34 | 35 | public String getMsg() { 36 | return msg; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /coordinator/src/main/java/com/tencent/rss/coordinator/AccessChecker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.coordinator; 20 | 21 | import java.io.Closeable; 22 | 23 | /** 24 | * Interface for checking the access info from the client-side. 25 | */ 26 | public interface AccessChecker extends Closeable { 27 | 28 | /** 29 | * Called when the AccessManager handle the access request. 30 | * 31 | * @param accessInfo access info of the client 32 | * @return access check result 33 | */ 34 | AccessCheckResult check(AccessInfo accessInfo); 35 | } 36 | -------------------------------------------------------------------------------- /coordinator/src/main/java/com/tencent/rss/coordinator/AccessInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.coordinator; 20 | 21 | import java.util.Set; 22 | 23 | import com.google.common.collect.Sets; 24 | 25 | public class AccessInfo { 26 | private final String accessId; 27 | private final Set tags; 28 | 29 | public AccessInfo(String accessId, Set tags) { 30 | this.accessId = accessId; 31 | this.tags = tags; 32 | } 33 | 34 | public AccessInfo(String accessId) { 35 | this(accessId, Sets.newHashSet()); 36 | } 37 | 38 | public String getAccessId() { 39 | return accessId; 40 | } 41 | 42 | public Set getTags() { 43 | return tags; 44 | } 45 | 46 | @Override 47 | public String toString() { 48 | return "AccessInfo{" 49 | + "accessId='" + accessId + '\'' 50 | + ", tags=" + tags 51 | + '}'; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /coordinator/src/main/java/com/tencent/rss/coordinator/AssignmentStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.coordinator; 20 | 21 | import java.util.Set; 22 | 23 | public interface AssignmentStrategy { 24 | 25 | PartitionRangeAssignment assign(int totalPartitionNum, int partitionNumPerRange, 26 | int replica, Set requiredTags); 27 | 28 | } 29 | -------------------------------------------------------------------------------- /coordinator/src/main/java/com/tencent/rss/coordinator/AssignmentStrategyFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.coordinator; 20 | 21 | public class AssignmentStrategyFactory { 22 | 23 | private CoordinatorConf conf; 24 | private ClusterManager clusterManager; 25 | 26 | public AssignmentStrategyFactory(CoordinatorConf conf, ClusterManager clusterManager) { 27 | this.conf = conf; 28 | this.clusterManager = clusterManager; 29 | } 30 | 31 | public AssignmentStrategy getAssignmentStrategy() { 32 | String strategy = conf.getString(CoordinatorConf.COORDINATOR_ASSIGNMENT_STRATEGY); 33 | if (StrategyName.BASIC.name().equals(strategy)) { 34 | return new BasicAssignmentStrategy(clusterManager); 35 | } else if (StrategyName.PARTITION_BALANCE.name().equals(strategy)) { 36 | return new PartitionBalanceAssignmentStrategy(clusterManager); 37 | } else { 38 | throw new UnsupportedOperationException("Unsupported assignment strategy."); 39 | } 40 | } 41 | 42 | private enum StrategyName { 43 | BASIC, 44 | PARTITION_BALANCE 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /coordinator/src/main/java/com/tencent/rss/coordinator/ClusterManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.coordinator; 20 | 21 | import java.io.Closeable; 22 | import java.util.List; 23 | import java.util.Set; 24 | 25 | public interface ClusterManager extends Closeable { 26 | 27 | /** 28 | * Add a server to the cluster. 29 | * 30 | * @param shuffleServerInfo server info 31 | */ 32 | void add(ServerNode shuffleServerInfo); 33 | 34 | /** 35 | * Get available nodes from the cluster 36 | * 37 | * @param requiredTags tags for filter 38 | * @return list of available server nodes 39 | */ 40 | List getServerList(Set requiredTags); 41 | 42 | /** 43 | * @return number of server nodes in the cluster 44 | */ 45 | int getNodesNum(); 46 | 47 | /** 48 | * @return list all server nodes in the cluster 49 | */ 50 | List list(); 51 | 52 | int getShuffleNodesMax(); 53 | } 54 | -------------------------------------------------------------------------------- /coordinator/src/main/java/com/tencent/rss/coordinator/ClusterManagerFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.coordinator; 20 | 21 | import org.apache.hadoop.conf.Configuration; 22 | 23 | public class ClusterManagerFactory { 24 | 25 | CoordinatorConf conf; 26 | Configuration hadoopConf; 27 | 28 | public ClusterManagerFactory(CoordinatorConf conf, Configuration hadoopConf) { 29 | this.conf = conf; 30 | this.hadoopConf = hadoopConf; 31 | } 32 | 33 | public ClusterManager getClusterManager() throws Exception { 34 | return new SimpleClusterManager(conf, hadoopConf); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /coordinator/src/main/java/com/tencent/rss/coordinator/CoordinatorFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.coordinator; 20 | 21 | import com.tencent.rss.common.rpc.GrpcServer; 22 | import com.tencent.rss.common.rpc.ServerInterface; 23 | 24 | public class CoordinatorFactory { 25 | 26 | private final CoordinatorServer coordinatorServer; 27 | private final CoordinatorConf conf; 28 | 29 | public CoordinatorFactory(CoordinatorServer coordinatorServer) { 30 | this.coordinatorServer = coordinatorServer; 31 | this.conf = coordinatorServer.getCoordinatorConf(); 32 | } 33 | 34 | public ServerInterface getServer() { 35 | String type = conf.getString(CoordinatorConf.RPC_SERVER_TYPE); 36 | if (type.equals(ServerType.GRPC.name())) { 37 | return new GrpcServer(conf, new CoordinatorGrpcService(coordinatorServer), 38 | coordinatorServer.getGrpcMetrics()); 39 | } else { 40 | throw new UnsupportedOperationException("Unsupported server type " + type); 41 | } 42 | } 43 | 44 | enum ServerType { 45 | GRPC 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /coordinator/src/main/java/com/tencent/rss/coordinator/CoordinatorGrpcMetrics.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.coordinator; 20 | 21 | import com.tencent.rss.common.metrics.GRPCMetrics; 22 | 23 | public class CoordinatorGrpcMetrics extends GRPCMetrics { 24 | 25 | public static final String HEARTBEAT_METHOD = "heartbeat"; 26 | public static final String GET_SHUFFLE_ASSIGNMENTS_METHOD = "getShuffleAssignments"; 27 | 28 | private static final String GRPC_OPEN = "grpc_open"; 29 | private static final String GRPC_TOTAL = "grpc_total"; 30 | private static final String GRPC_GET_SHUFFLE_ASSIGNMENTS = "grpc_get_shuffle_assignments"; 31 | private static final String GRPC_HEARTBEAT = "grpc_heartbeat"; 32 | private static final String GRPC_GET_SHUFFLE_ASSIGNMENTS_TOTAL = 33 | "grpc_get_shuffle_assignments_total"; 34 | private static final String GRPC_HEARTBEAT_TOTAL = 35 | "grpc_heartbeat_total"; 36 | 37 | @Override 38 | public void registerMetrics() { 39 | gaugeGrpcOpen = metricsManager.addGauge(GRPC_OPEN); 40 | counterGrpcTotal = metricsManager.addCounter(GRPC_TOTAL); 41 | gaugeMap.putIfAbsent(HEARTBEAT_METHOD, 42 | metricsManager.addGauge(GRPC_HEARTBEAT)); 43 | gaugeMap.putIfAbsent(GET_SHUFFLE_ASSIGNMENTS_METHOD, 44 | metricsManager.addGauge(GRPC_GET_SHUFFLE_ASSIGNMENTS)); 45 | counterMap.putIfAbsent(HEARTBEAT_METHOD, 46 | metricsManager.addCounter(GRPC_HEARTBEAT_TOTAL)); 47 | counterMap.putIfAbsent(GET_SHUFFLE_ASSIGNMENTS_METHOD, 48 | metricsManager.addCounter(GRPC_GET_SHUFFLE_ASSIGNMENTS_TOTAL)); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /coordinator/src/test/java/com/tencent/rss/coordinator/CoordinatorConfTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.coordinator; 20 | 21 | import static org.junit.jupiter.api.Assertions.assertEquals; 22 | 23 | import java.util.Objects; 24 | import org.junit.jupiter.api.Test; 25 | 26 | public class CoordinatorConfTest { 27 | 28 | @Test 29 | public void test() { 30 | final String filePath = Objects.requireNonNull( 31 | getClass().getClassLoader().getResource("coordinator.conf")).getFile(); 32 | CoordinatorConf conf = new CoordinatorConf(filePath); 33 | 34 | // test base conf 35 | assertEquals(9527, conf.getInteger(CoordinatorConf.RPC_SERVER_PORT)); 36 | assertEquals("testRpc", conf.getString(CoordinatorConf.RPC_SERVER_TYPE)); 37 | assertEquals(9526, conf.getInteger(CoordinatorConf.JETTY_HTTP_PORT)); 38 | 39 | // test coordinator specific conf 40 | assertEquals("/a/b/c", conf.getString(CoordinatorConf.COORDINATOR_EXCLUDE_NODES_FILE_PATH)); 41 | assertEquals(37, conf.getInteger(CoordinatorConf.COORDINATOR_SHUFFLE_NODES_MAX)); 42 | assertEquals(123, conf.getLong(CoordinatorConf.COORDINATOR_HEARTBEAT_TIMEOUT)); 43 | 44 | // test default conf 45 | assertEquals("PARTITION_BALANCE", conf.getString(CoordinatorConf.COORDINATOR_ASSIGNMENT_STRATEGY)); 46 | assertEquals(256, conf.getInteger(CoordinatorConf.JETTY_CORE_POOL_SIZE)); 47 | assertEquals(60 * 1000, conf.getLong(CoordinatorConf.COORDINATOR_EXCLUDE_NODES_CHECK_INTERVAL)); 48 | 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /coordinator/src/test/java/com/tencent/rss/coordinator/PartitionRangeTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.coordinator; 20 | 21 | import static org.junit.jupiter.api.Assertions.assertEquals; 22 | import static org.junit.jupiter.api.Assertions.assertFalse; 23 | 24 | import com.tencent.rss.common.PartitionRange; 25 | import org.junit.jupiter.api.Test; 26 | 27 | public class PartitionRangeTest { 28 | 29 | @Test 30 | public void test() { 31 | PartitionRange range1 = new PartitionRange(0, 5); 32 | PartitionRange range2 = new PartitionRange(0, 5); 33 | assertFalse(range1 == range2); 34 | assertEquals(range1, range2); 35 | assertEquals(0, range1.getStart()); 36 | assertEquals(5, range1.getEnd()); 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /coordinator/src/test/java/com/tencent/rss/coordinator/ServerNodeTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.coordinator; 20 | 21 | import static org.junit.jupiter.api.Assertions.assertEquals; 22 | 23 | import com.google.common.collect.Lists; 24 | import com.google.common.collect.Sets; 25 | import java.util.Collections; 26 | import java.util.List; 27 | import java.util.Set; 28 | import org.junit.jupiter.api.Test; 29 | 30 | public class ServerNodeTest { 31 | 32 | @Test 33 | public void compareTest() { 34 | Set tags = Sets.newHashSet("test"); 35 | ServerNode sn1 = new ServerNode("sn1", "ip", 0, 100L, 50L, 20, 36 | 10, tags, true); 37 | ServerNode sn2 = new ServerNode("sn2", "ip", 0, 100L, 50L, 21, 38 | 10, tags, true); 39 | ServerNode sn3 = new ServerNode("sn3", "ip", 0, 100L, 50L, 20, 40 | 11, tags, true); 41 | List nodes = Lists.newArrayList(sn1, sn2, sn3); 42 | Collections.sort(nodes); 43 | assertEquals("sn2", nodes.get(0).getId()); 44 | assertEquals("sn1", nodes.get(1).getId()); 45 | assertEquals("sn3", nodes.get(2).getId()); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /coordinator/src/test/resources/coordinator.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Tencent is pleased to support the open source community by making 3 | # Firestorm-Spark remote shuffle server available. 4 | # 5 | # Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | # this file except in compliance with the License. You may obtain a copy of the 9 | # License at 10 | # 11 | # https://opensource.org/licenses/Apache-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | # WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations under the License. 17 | # 18 | 19 | # base conf 20 | rss.rpc.server.port 9527 21 | rss.rpc.server.type testRpc 22 | rss.jetty.http.port 9526 23 | 24 | # coordinator specific conf 25 | rss.coordinator.exclude.nodes.file.path /a/b/c 26 | rss.coordinator.shuffle.nodes.max 37 27 | rss.coordinator.server.heartbeat.timeout 123 28 | rss.coordinator.access.candidates.updateIntervalSec 1 29 | rss.coordinator.access.loadChecker.serverNum.threshold 2 30 | rss.coordinator.access.loadChecker.memory.percentage 20.0 31 | rss.coordinator.dynamicClientConf.updateIntervalSec 1 32 | -------------------------------------------------------------------------------- /coordinator/src/test/resources/empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/Firestorm/c74cc848329e013694796097a8b5c6416a3929ac/coordinator/src/test/resources/empty -------------------------------------------------------------------------------- /docs/_config.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Tencent is pleased to support the open source community by making 3 | # Firestorm-Spark remote shuffle server available. 4 | # 5 | # Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | # this file except in compliance with the License. You may obtain a copy of the 9 | # License at 10 | # 11 | # https://opensource.org/licenses/Apache-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | # WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations under the License. 17 | # 18 | 19 | title: Firestorm homepage 20 | baseurl: "" 21 | url: "" 22 | 23 | # Build settings 24 | theme: minima 25 | 26 | kramdown: 27 | math_engine: mathjax 28 | syntax_highlighter: rouge 29 | -------------------------------------------------------------------------------- /docs/asset/rss_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/Firestorm/c74cc848329e013694796097a8b5c6416a3929ac/docs/asset/rss_architecture.png -------------------------------------------------------------------------------- /docs/asset/rss_data_format.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/Firestorm/c74cc848329e013694796097a8b5c6416a3929ac/docs/asset/rss_data_format.png -------------------------------------------------------------------------------- /docs/asset/rss_shuffle_write.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/Firestorm/c74cc848329e013694796097a8b5c6416a3929ac/docs/asset/rss_shuffle_write.png -------------------------------------------------------------------------------- /docs/asset/wechat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tencent/Firestorm/c74cc848329e013694796097a8b5c6416a3929ac/docs/asset/wechat.png -------------------------------------------------------------------------------- /docs/coordinator_guide.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | displayTitle: Firestorm Coordinator Guide 4 | title: Firestorm Coordinator Guide 5 | description: Firestorm Coordinator Guide 6 | --- 7 | 8 | # Firestorm Coordinator Guide -------------------------------------------------------------------------------- /docs/gen-doc.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | # 4 | # Tencent is pleased to support the open source community by making 5 | # Firestorm-Spark remote shuffle server available. 6 | # 7 | # Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use 10 | # this file except in compliance with the License. You may obtain a copy of the 11 | # License at 12 | # 13 | # https://opensource.org/licenses/Apache-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 17 | # WARRANTIES OF ANY KIND, either express or implied. See the License for the 18 | # specific language governing permissions and limitations under the License. 19 | # 20 | 21 | set -e 22 | 23 | BASE_DIR="$(cd "`dirname "$0"`/.."; pwd)" 24 | cd "${BASE_DIR}" 25 | mvn clean javadoc:aggregate 26 | mv target/site/apidocs docs 27 | cd docs 28 | 29 | jekyll build 30 | 31 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | displayTitle: Firestorm Overview 4 | title: Overview 5 | description: Firestorm documentation homepage 6 | --- 7 | 8 | Firestorm is a Remote Shuffle Service, and provides the capability for Apache Spark applications 9 | to store shuffle data on remote servers. 10 | 11 | 12 | ![Rss Architecture](asset/rss_architecture.png) 13 | 14 | 15 | More advanced details for Firestorm users are available in the following: 16 | 17 | - [Firestorm Coordinator Guide](coordinator_guide.html) 18 | 19 | - [Firestorm Shuffle Server Guide](server_guide.html) 20 | 21 | - [Firestorm Shuffle Client Guide](client_guide.html) 22 | 23 | 24 | Here you can read API docs for Firestorm along with its submodules. 25 | 26 | - [Java API (Javadoc)](apidocs/index.html) -------------------------------------------------------------------------------- /docs/server_guide.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | displayTitle: Firestorm Shuffle Server Guide 4 | title: Firestorm Shuffle Server Guide 5 | description: Firestorm Shuffle Server Guide 6 | --- 7 | # Firestorm Shuffle Server Guide -------------------------------------------------------------------------------- /integration-test/common/src/test/java/com/tencent/rss/test/CoordinatorTestBase.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | * this file except in compliance with the License. You may obtain a copy of the 8 | * License at 9 | * 10 | * https://opensource.org/licenses/Apache-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations under the License. 16 | */ 17 | 18 | package com.tencent.rss.test; 19 | 20 | import com.tencent.rss.client.factory.CoordinatorClientFactory; 21 | import com.tencent.rss.client.impl.grpc.CoordinatorGrpcClient; 22 | import org.junit.jupiter.api.AfterEach; 23 | import org.junit.jupiter.api.BeforeEach; 24 | 25 | public class CoordinatorTestBase extends IntegrationTestBase { 26 | 27 | protected CoordinatorClientFactory factory = new CoordinatorClientFactory("GRPC"); 28 | protected CoordinatorGrpcClient coordinatorClient; 29 | @BeforeEach 30 | public void createClient() { 31 | coordinatorClient = 32 | (CoordinatorGrpcClient) factory.createCoordinatorClient(LOCALHOST, COORDINATOR_PORT_1); 33 | } 34 | 35 | @AfterEach 36 | public void closeClient() { 37 | if (coordinatorClient != null) { 38 | coordinatorClient.close(); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /integration-test/common/src/test/java/com/tencent/rss/test/CoordinatorTestUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | * this file except in compliance with the License. You may obtain a copy of the 8 | * License at 9 | * 10 | * https://opensource.org/licenses/Apache-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations under the License. 16 | */ 17 | 18 | package com.tencent.rss.test; 19 | 20 | import com.tencent.rss.client.impl.grpc.CoordinatorGrpcClient; 21 | import com.tencent.rss.proto.RssProtos; 22 | 23 | public class CoordinatorTestUtils { 24 | 25 | private CoordinatorTestUtils() { 26 | } 27 | 28 | public static void waitForRegister(CoordinatorGrpcClient coordinatorClient, int expectedServers) throws Exception { 29 | RssProtos.GetShuffleServerListResponse response; 30 | int count = 0; 31 | do { 32 | response = coordinatorClient.getShuffleServerList(); 33 | Thread.sleep(1000); 34 | if (count > 10) { 35 | throw new RuntimeException("No shuffle server connected"); 36 | } 37 | count++; 38 | } while (response.getServersCount() < expectedServers); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /integration-test/mr/src/test/java/com/tencent/rss/test/FailoverAppMaster.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | package com.tencent.rss.test; 19 | 20 | import org.apache.hadoop.mapreduce.RssMRUtils; 21 | import org.apache.hadoop.mapreduce.v2.app.RssMRAppMaster; 22 | import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; 23 | 24 | public class FailoverAppMaster { 25 | public static void main(String[] args) { 26 | ApplicationAttemptId applicationAttemptId = RssMRUtils.getApplicationAttemptId(); 27 | if (applicationAttemptId.getAttemptId() == 1) { 28 | System.exit(1); 29 | } 30 | RssMRAppMaster.main(args); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /integration-test/spark-common/src/test/java/com/tencent/rss/test/CombineByKeyTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.test; 20 | 21 | import java.util.Map; 22 | import org.apache.spark.api.java.JavaPairRDD; 23 | import org.apache.spark.api.java.JavaSparkContext; 24 | import org.apache.spark.sql.SparkSession; 25 | import org.junit.jupiter.api.Test; 26 | import scala.Tuple2; 27 | 28 | public class CombineByKeyTest extends SimpleTestBase { 29 | 30 | @Test 31 | public void combineByKeyTest() throws Exception { 32 | run(); 33 | } 34 | 35 | @Override 36 | public Map runTest(SparkSession spark, String fileName) throws Exception { 37 | // take a rest to make sure shuffle server is registered 38 | Thread.sleep(4000); 39 | JavaSparkContext jsc = new JavaSparkContext(spark.sparkContext()); 40 | JavaPairRDD> javaPairRDD = TestUtils.combineByKeyRDD( 41 | TestUtils.getRDD(jsc)); 42 | return javaPairRDD.collectAsMap(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /integration-test/spark-common/src/test/java/com/tencent/rss/test/GroupByKeyTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.test; 20 | 21 | import com.google.common.collect.Lists; 22 | 23 | import java.util.Map; 24 | 25 | import org.apache.spark.api.java.JavaPairRDD; 26 | import org.apache.spark.api.java.JavaSparkContext; 27 | import org.apache.spark.sql.SparkSession; 28 | import org.junit.jupiter.api.Test; 29 | import scala.Tuple2; 30 | 31 | public class GroupByKeyTest extends SimpleTestBase { 32 | 33 | @Test 34 | public void groupByTest() throws Exception { 35 | run(); 36 | } 37 | 38 | @Override 39 | public Map runTest(SparkSession spark, String fileName) throws Exception { 40 | // take a rest to make sure shuffle server is registered 41 | Thread.sleep(3000); 42 | JavaSparkContext jsc = new JavaSparkContext(spark.sparkContext()); 43 | JavaPairRDD javaPairRDD1 = jsc.parallelizePairs(Lists.newArrayList( 44 | new Tuple2<>("a", "1"), new Tuple2<>("b", "2"), 45 | new Tuple2<>("c", "3"), new Tuple2<>("d", "4")), 2); 46 | JavaPairRDD> javaPairRDD = javaPairRDD1.groupByKey().sortByKey(); 47 | return javaPairRDD.collectAsMap(); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /integration-test/spark-common/src/test/java/com/tencent/rss/test/SimpleTestBase.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.test; 20 | 21 | import java.util.Map; 22 | 23 | import com.google.common.collect.Maps; 24 | import org.apache.spark.SparkConf; 25 | import org.apache.spark.shuffle.RssSparkConfig; 26 | import org.junit.jupiter.api.BeforeAll; 27 | 28 | import com.tencent.rss.coordinator.CoordinatorConf; 29 | import com.tencent.rss.server.ShuffleServerConf; 30 | import com.tencent.rss.storage.util.StorageType; 31 | 32 | public abstract class SimpleTestBase extends SparkIntegrationTestBase { 33 | @BeforeAll 34 | public static void setupServers() throws Exception { 35 | CoordinatorConf coordinatorConf = getCoordinatorConf(); 36 | Map dynamicConf = Maps.newHashMap(); 37 | dynamicConf.put(CoordinatorConf.COORDINATOR_REMOTE_STORAGE_PATH.key(), HDFS_URI + "rss/test"); 38 | dynamicConf.put(RssSparkConfig.RSS_STORAGE_TYPE, StorageType.MEMORY_LOCALFILE_HDFS.name()); 39 | addDynamicConf(coordinatorConf, dynamicConf); 40 | createCoordinatorServer(coordinatorConf); 41 | ShuffleServerConf shuffleServerConf = getShuffleServerConf(); 42 | createShuffleServer(shuffleServerConf); 43 | startServers(); 44 | } 45 | 46 | @Override 47 | public void updateSparkConfCustomer(SparkConf sparkConf) { 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /integration-test/spark-common/src/test/java/com/tencent/rss/test/TestUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.test; 20 | 21 | import com.google.common.collect.Lists; 22 | import org.apache.spark.api.java.JavaPairRDD; 23 | import org.apache.spark.api.java.JavaSparkContext; 24 | import org.apache.spark.api.java.function.Function; 25 | import org.apache.spark.api.java.function.Function2; 26 | import scala.Tuple2; 27 | 28 | public class TestUtils { 29 | private TestUtils() { 30 | 31 | } 32 | static JavaPairRDD getRDD(JavaSparkContext jsc) { 33 | JavaPairRDD javaPairRDD1 = jsc.parallelizePairs(Lists.newArrayList( 34 | new Tuple2<>("cat1", 11), new Tuple2<>("dog", 22), 35 | new Tuple2<>("cat", 33), new Tuple2<>("pig", 44), 36 | new Tuple2<>("duck", 55), new Tuple2<>("cat", 66)), 2); 37 | return javaPairRDD1; 38 | } 39 | 40 | static JavaPairRDD> combineByKeyRDD(JavaPairRDD javaPairRDD1) { 41 | JavaPairRDD> javaPairRDD = javaPairRDD1 42 | .combineByKey((Function>) i -> new Tuple2<>(1, i), 43 | (Function2, Integer, Tuple2>) (tuple, i) -> 44 | new Tuple2<>(tuple._1 + 1, tuple._2 + i), 45 | (Function2, Tuple2, Tuple2>) (tuple1, tuple2) -> 46 | new Tuple2<>(tuple1._1 + tuple2._1, tuple1._2 + tuple2._2) 47 | ); 48 | return javaPairRDD; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /integration-test/spark-common/src/test/resources/candidates: -------------------------------------------------------------------------------- 1 | # 2 | # Tencent is pleased to support the open source community by making 3 | # Firestorm-Spark remote shuffle server available. 4 | # 5 | # Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | # this file except in compliance with the License. You may obtain a copy of the 9 | # License at 10 | # 11 | # https://opensource.org/licenses/Apache-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | # WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations under the License. 17 | # 18 | 19 | test_access_id -------------------------------------------------------------------------------- /internal-client/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 23 | 4.0.0 24 | 25 | com.tencent.rss 26 | rss-main 27 | 0.6.0-snapshot 28 | 29 | 30 | com.tencent.rss 31 | rss-internal-client 32 | 0.6.0-snapshot 33 | jar 34 | 35 | 36 | 37 | com.tencent.rss 38 | rss-common 39 | 40 | 41 | org.slf4j 42 | slf4j-api 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/request/RssAccessClusterRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.request; 20 | 21 | import java.util.Set; 22 | 23 | public class RssAccessClusterRequest { 24 | 25 | private final String accessId; 26 | private final Set tags; 27 | private final int timeoutMs; 28 | 29 | public RssAccessClusterRequest(String accessId, Set tags, int timeoutMs) { 30 | this.accessId = accessId; 31 | this.tags = tags; 32 | this.timeoutMs = timeoutMs; 33 | } 34 | 35 | public String getAccessId() { 36 | return accessId; 37 | } 38 | 39 | public Set getTags() { 40 | return tags; 41 | } 42 | 43 | public int getTimeoutMs() { 44 | return timeoutMs; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/request/RssAppHeartBeatRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.request; 20 | 21 | public class RssAppHeartBeatRequest { 22 | 23 | private final String appId; 24 | private final long timeoutMs; 25 | 26 | public RssAppHeartBeatRequest(String appId, long timeoutMs) { 27 | this.appId = appId; 28 | this.timeoutMs = timeoutMs; 29 | } 30 | 31 | public String getAppId() { 32 | return appId; 33 | } 34 | 35 | public long getTimeoutMs() { 36 | return timeoutMs; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/request/RssFetchClientConfRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.request; 20 | 21 | public class RssFetchClientConfRequest { 22 | private final int timeoutMs; 23 | 24 | public RssFetchClientConfRequest(int timeoutMs) { 25 | this.timeoutMs = timeoutMs; 26 | } 27 | 28 | public int getTimeoutMs() { 29 | return timeoutMs; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/request/RssFetchRemoteStorageRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.request; 20 | 21 | public class RssFetchRemoteStorageRequest { 22 | 23 | private String appId; 24 | 25 | public RssFetchRemoteStorageRequest(String appId) { 26 | this.appId = appId; 27 | } 28 | 29 | public String getAppId() { 30 | return appId; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/request/RssFinishShuffleRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.request; 20 | 21 | public class RssFinishShuffleRequest { 22 | 23 | private String appId; 24 | private int shuffleId; 25 | 26 | public RssFinishShuffleRequest(String appId, int shuffleId) { 27 | this.appId = appId; 28 | this.shuffleId = shuffleId; 29 | } 30 | 31 | public String getAppId() { 32 | return appId; 33 | } 34 | 35 | public int getShuffleId() { 36 | return shuffleId; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/request/RssGetInMemoryShuffleDataRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.request; 20 | 21 | public class RssGetInMemoryShuffleDataRequest { 22 | private final String appId; 23 | private final int shuffleId; 24 | private final int partitionId; 25 | private final long lastBlockId; 26 | private final int readBufferSize; 27 | 28 | public RssGetInMemoryShuffleDataRequest( 29 | String appId, int shuffleId, int partitionId, long lastBlockId, int readBufferSize) { 30 | this.appId = appId; 31 | this.shuffleId = shuffleId; 32 | this.partitionId = partitionId; 33 | this.lastBlockId = lastBlockId; 34 | this.readBufferSize = readBufferSize; 35 | } 36 | 37 | public String getAppId() { 38 | return appId; 39 | } 40 | 41 | public int getShuffleId() { 42 | return shuffleId; 43 | } 44 | 45 | public int getPartitionId() { 46 | return partitionId; 47 | } 48 | 49 | public long getLastBlockId() { 50 | return lastBlockId; 51 | } 52 | 53 | public int getReadBufferSize() { 54 | return readBufferSize; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/request/RssGetShuffleAssignmentsRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.request; 20 | 21 | import java.util.Set; 22 | 23 | public class RssGetShuffleAssignmentsRequest { 24 | 25 | private String appId; 26 | private int shuffleId; 27 | private int partitionNum; 28 | private int partitionNumPerRange; 29 | private int dataReplica; 30 | private Set requiredTags; 31 | 32 | public RssGetShuffleAssignmentsRequest(String appId, int shuffleId, int partitionNum, 33 | int partitionNumPerRange, int dataReplica, Set requiredTags) { 34 | this.appId = appId; 35 | this.shuffleId = shuffleId; 36 | this.partitionNum = partitionNum; 37 | this.partitionNumPerRange = partitionNumPerRange; 38 | this.dataReplica = dataReplica; 39 | this.requiredTags = requiredTags; 40 | } 41 | 42 | public String getAppId() { 43 | return appId; 44 | } 45 | 46 | public int getShuffleId() { 47 | return shuffleId; 48 | } 49 | 50 | public int getPartitionNum() { 51 | return partitionNum; 52 | } 53 | 54 | public int getPartitionNumPerRange() { 55 | return partitionNumPerRange; 56 | } 57 | 58 | public int getDataReplica() { 59 | return dataReplica; 60 | } 61 | 62 | public Set getRequiredTags() { 63 | return requiredTags; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/request/RssGetShuffleDataRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.request; 20 | 21 | public class RssGetShuffleDataRequest { 22 | 23 | private final String appId; 24 | private final int shuffleId; 25 | private final int partitionId; 26 | private final int partitionNumPerRange; 27 | private final int partitionNum; 28 | private final long offset; 29 | private final int length; 30 | 31 | public RssGetShuffleDataRequest(String appId, int shuffleId, int partitionId, int partitionNumPerRange, 32 | int partitionNum, long offset, int length) { 33 | this.appId = appId; 34 | this.shuffleId = shuffleId; 35 | this.partitionId = partitionId; 36 | this.partitionNumPerRange = partitionNumPerRange; 37 | this.partitionNum = partitionNum; 38 | this.offset = offset; 39 | this.length = length; 40 | } 41 | 42 | public String getAppId() { 43 | return appId; 44 | } 45 | 46 | public int getShuffleId() { 47 | return shuffleId; 48 | } 49 | 50 | public int getPartitionId() { 51 | return partitionId; 52 | } 53 | 54 | public int getPartitionNumPerRange() { 55 | return partitionNumPerRange; 56 | } 57 | 58 | public int getPartitionNum() { 59 | return partitionNum; 60 | } 61 | 62 | public long getOffset() { 63 | return offset; 64 | } 65 | 66 | public int getLength() { 67 | return length; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/request/RssGetShuffleIndexRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.request; 20 | 21 | public class RssGetShuffleIndexRequest { 22 | 23 | private final String appId; 24 | private final int shuffleId; 25 | private final int partitionId; 26 | private final int partitionNumPerRange; 27 | private final int partitionNum; 28 | 29 | public RssGetShuffleIndexRequest( 30 | String appId, 31 | int shuffleId, 32 | int partitionId, 33 | int partitionNumPerRange, 34 | int partitionNum) { 35 | this.appId = appId; 36 | this.shuffleId = shuffleId; 37 | this.partitionId = partitionId; 38 | this.partitionNumPerRange = partitionNumPerRange; 39 | this.partitionNum = partitionNum; 40 | } 41 | 42 | public String getAppId() { 43 | return appId; 44 | } 45 | 46 | public int getShuffleId() { 47 | return shuffleId; 48 | } 49 | 50 | public int getPartitionId() { 51 | return partitionId; 52 | } 53 | 54 | public int getPartitionNumPerRange() { 55 | return partitionNumPerRange; 56 | } 57 | 58 | public int getPartitionNum() { 59 | return partitionNum; 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/request/RssGetShuffleResultRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.request; 20 | 21 | public class RssGetShuffleResultRequest { 22 | 23 | private String appId; 24 | private int shuffleId; 25 | private int partitionId; 26 | 27 | public RssGetShuffleResultRequest(String appId, int shuffleId, int partitionId) { 28 | this.appId = appId; 29 | this.shuffleId = shuffleId; 30 | this.partitionId = partitionId; 31 | } 32 | 33 | public String getAppId() { 34 | return appId; 35 | } 36 | 37 | public int getShuffleId() { 38 | return shuffleId; 39 | } 40 | 41 | public int getPartitionId() { 42 | return partitionId; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/request/RssRegisterShuffleRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.request; 20 | 21 | import java.util.List; 22 | 23 | import com.tencent.rss.common.PartitionRange; 24 | import com.tencent.rss.common.RemoteStorageInfo; 25 | 26 | public class RssRegisterShuffleRequest { 27 | 28 | private String appId; 29 | private int shuffleId; 30 | private List partitionRanges; 31 | private RemoteStorageInfo remoteStorageInfo; 32 | 33 | public RssRegisterShuffleRequest( 34 | String appId, 35 | int shuffleId, 36 | List partitionRanges, 37 | RemoteStorageInfo remoteStorageInfo) { 38 | this.appId = appId; 39 | this.shuffleId = shuffleId; 40 | this.partitionRanges = partitionRanges; 41 | this.remoteStorageInfo = remoteStorageInfo; 42 | } 43 | 44 | public RssRegisterShuffleRequest( 45 | String appId, 46 | int shuffleId, 47 | List partitionRanges, 48 | String remoteStoragePath) { 49 | this(appId, shuffleId, partitionRanges, new RemoteStorageInfo(remoteStoragePath)); 50 | } 51 | 52 | public String getAppId() { 53 | return appId; 54 | } 55 | 56 | public int getShuffleId() { 57 | return shuffleId; 58 | } 59 | 60 | public List getPartitionRanges() { 61 | return partitionRanges; 62 | } 63 | 64 | public RemoteStorageInfo getRemoteStorageInfo() { 65 | return remoteStorageInfo; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/request/RssReportShuffleResultRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.request; 20 | 21 | import java.util.List; 22 | import java.util.Map; 23 | 24 | public class RssReportShuffleResultRequest { 25 | 26 | private String appId; 27 | private int shuffleId; 28 | private long taskAttemptId; 29 | private int bitmapNum; 30 | private Map> partitionToBlockIds; 31 | 32 | public RssReportShuffleResultRequest(String appId, int shuffleId, long taskAttemptId, 33 | Map> partitionToBlockIds, int bitmapNum) { 34 | this.appId = appId; 35 | this.shuffleId = shuffleId; 36 | this.taskAttemptId = taskAttemptId; 37 | this.bitmapNum = bitmapNum; 38 | this.partitionToBlockIds = partitionToBlockIds; 39 | } 40 | 41 | public String getAppId() { 42 | return appId; 43 | } 44 | 45 | public int getShuffleId() { 46 | return shuffleId; 47 | } 48 | 49 | public long getTaskAttemptId() { 50 | return taskAttemptId; 51 | } 52 | 53 | public int getBitmapNum() { 54 | return bitmapNum; 55 | } 56 | 57 | public Map> getPartitionToBlockIds() { 58 | return partitionToBlockIds; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/request/RssSendCommitRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.request; 20 | 21 | public class RssSendCommitRequest { 22 | 23 | private String appId; 24 | private int shuffleId; 25 | 26 | public RssSendCommitRequest(String appId, int shuffleId) { 27 | this.appId = appId; 28 | this.shuffleId = shuffleId; 29 | } 30 | 31 | public String getAppId() { 32 | return appId; 33 | } 34 | 35 | public int getShuffleId() { 36 | return shuffleId; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/request/RssSendShuffleDataRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.request; 20 | 21 | import java.util.List; 22 | import java.util.Map; 23 | 24 | import com.tencent.rss.common.ShuffleBlockInfo; 25 | 26 | public class RssSendShuffleDataRequest { 27 | 28 | private String appId; 29 | private int retryMax; 30 | private long retryIntervalMax; 31 | private Map>> shuffleIdToBlocks; 32 | 33 | public RssSendShuffleDataRequest(String appId, int retryMax, long retryIntervalMax, 34 | Map>> shuffleIdToBlocks) { 35 | this.appId = appId; 36 | this.retryMax = retryMax; 37 | this.retryIntervalMax = retryIntervalMax; 38 | this.shuffleIdToBlocks = shuffleIdToBlocks; 39 | } 40 | 41 | public String getAppId() { 42 | return appId; 43 | } 44 | 45 | public int getRetryMax() { 46 | return retryMax; 47 | } 48 | 49 | public long getRetryIntervalMax() { 50 | return retryIntervalMax; 51 | } 52 | 53 | public Map>> getShuffleIdToBlocks() { 54 | return shuffleIdToBlocks; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/response/ClientResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.response; 20 | 21 | public class ClientResponse { 22 | 23 | private final ResponseStatusCode statusCode; 24 | private final String message; 25 | 26 | public ClientResponse(ResponseStatusCode statusCode) { 27 | this.statusCode = statusCode; 28 | this.message = ""; 29 | } 30 | 31 | public ClientResponse(ResponseStatusCode statusCode, String message) { 32 | this.statusCode = statusCode; 33 | this.message = message; 34 | } 35 | 36 | public ResponseStatusCode getStatusCode() { 37 | return statusCode; 38 | } 39 | 40 | public String getMessage() { 41 | return message; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/response/ResponseStatusCode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.response; 20 | 21 | public enum ResponseStatusCode { 22 | SUCCESS, 23 | DOUBLE_REGISTER, 24 | NO_BUFFER, 25 | INVALID_STORAGE, 26 | NO_REGISTER, 27 | NO_PARTITION, 28 | INTERNAL_ERROR, 29 | TIMEOUT, 30 | ACCESS_DENIED 31 | } 32 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/response/RssAccessClusterResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.response; 20 | 21 | public class RssAccessClusterResponse extends ClientResponse { 22 | 23 | public RssAccessClusterResponse(ResponseStatusCode statusCode, String messge) { 24 | super(statusCode, messge); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/response/RssAppHeartBeatResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.response; 20 | 21 | public class RssAppHeartBeatResponse extends ClientResponse { 22 | 23 | public RssAppHeartBeatResponse(ResponseStatusCode statusCode) { 24 | super(statusCode); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/response/RssFetchClientConfResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.response; 20 | 21 | import java.util.Map; 22 | 23 | import com.google.common.collect.Maps; 24 | 25 | public class RssFetchClientConfResponse extends ClientResponse { 26 | private final Map clientConf; 27 | 28 | public RssFetchClientConfResponse(ResponseStatusCode statusCode, String message) { 29 | this(statusCode, message, Maps.newHashMap()); 30 | } 31 | 32 | public RssFetchClientConfResponse( 33 | ResponseStatusCode statusCode, 34 | String message, 35 | Map clientConf) { 36 | super(statusCode, message); 37 | this.clientConf = clientConf; 38 | } 39 | 40 | public Map getClientConf() { 41 | return clientConf; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/response/RssFetchRemoteStorageResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.response; 20 | 21 | import com.tencent.rss.common.RemoteStorageInfo; 22 | 23 | public class RssFetchRemoteStorageResponse extends ClientResponse { 24 | private RemoteStorageInfo remoteStorageInfo; 25 | 26 | public RssFetchRemoteStorageResponse( 27 | ResponseStatusCode statusCode, 28 | RemoteStorageInfo remoteStorageInfo) { 29 | super(statusCode); 30 | this.remoteStorageInfo = remoteStorageInfo; 31 | } 32 | 33 | public RemoteStorageInfo getRemoteStorageInfo() { 34 | return remoteStorageInfo; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/response/RssFinishShuffleResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.response; 20 | 21 | public class RssFinishShuffleResponse extends ClientResponse { 22 | 23 | public RssFinishShuffleResponse(ResponseStatusCode statusCode) { 24 | super(statusCode); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/response/RssGetInMemoryShuffleDataResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.response; 20 | 21 | import java.util.List; 22 | 23 | import com.tencent.rss.common.BufferSegment; 24 | 25 | public class RssGetInMemoryShuffleDataResponse extends ClientResponse { 26 | 27 | private final byte[] data; 28 | private final List bufferSegments; 29 | 30 | public RssGetInMemoryShuffleDataResponse( 31 | ResponseStatusCode statusCode, byte[] data, List bufferSegments) { 32 | super(statusCode); 33 | this.bufferSegments = bufferSegments; 34 | this.data = data; 35 | } 36 | 37 | public byte[] getData() { 38 | return data; 39 | } 40 | 41 | public List getBufferSegments() { 42 | return bufferSegments; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/response/RssGetShuffleAssignmentsResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.response; 20 | 21 | import java.util.List; 22 | import java.util.Map; 23 | 24 | import com.tencent.rss.common.PartitionRange; 25 | import com.tencent.rss.common.ShuffleServerInfo; 26 | 27 | public class RssGetShuffleAssignmentsResponse extends ClientResponse { 28 | 29 | private Map> partitionToServers; 30 | private Map> serverToPartitionRanges; 31 | 32 | public RssGetShuffleAssignmentsResponse(ResponseStatusCode statusCode) { 33 | super(statusCode); 34 | } 35 | 36 | public Map> getPartitionToServers() { 37 | return partitionToServers; 38 | } 39 | 40 | public void setPartitionToServers( 41 | Map> partitionToServers) { 42 | this.partitionToServers = partitionToServers; 43 | } 44 | 45 | public Map> getServerToPartitionRanges() { 46 | return serverToPartitionRanges; 47 | } 48 | 49 | public void setServerToPartitionRanges( 50 | Map> serverToPartitionRanges) { 51 | this.serverToPartitionRanges = serverToPartitionRanges; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/response/RssGetShuffleDataResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.response; 20 | 21 | public class RssGetShuffleDataResponse extends ClientResponse { 22 | 23 | private final byte[] shuffleData; 24 | 25 | public RssGetShuffleDataResponse(ResponseStatusCode statusCode, byte[] data) { 26 | super(statusCode); 27 | this.shuffleData = data; 28 | } 29 | 30 | public byte[] getShuffleData() { 31 | return shuffleData; 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/response/RssGetShuffleIndexResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.response; 20 | 21 | import com.tencent.rss.common.ShuffleIndexResult; 22 | 23 | public class RssGetShuffleIndexResponse extends ClientResponse { 24 | private final ShuffleIndexResult shuffleIndexResult; 25 | 26 | public RssGetShuffleIndexResponse(ResponseStatusCode statusCode, byte[] data) { 27 | super(statusCode); 28 | this.shuffleIndexResult = new ShuffleIndexResult(data); 29 | } 30 | 31 | public ShuffleIndexResult getShuffleIndexResult() { 32 | return shuffleIndexResult; 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/response/RssGetShuffleResultResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.response; 20 | 21 | import java.io.IOException; 22 | 23 | import org.roaringbitmap.longlong.Roaring64NavigableMap; 24 | 25 | import com.tencent.rss.common.util.RssUtils; 26 | 27 | public class RssGetShuffleResultResponse extends ClientResponse { 28 | 29 | private Roaring64NavigableMap blockIdBitmap; 30 | 31 | public RssGetShuffleResultResponse(ResponseStatusCode statusCode, byte[] serializedBitmap) throws IOException { 32 | super(statusCode); 33 | blockIdBitmap = RssUtils.deserializeBitMap(serializedBitmap); 34 | } 35 | 36 | public Roaring64NavigableMap getBlockIdBitmap() { 37 | return blockIdBitmap; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/response/RssRegisterShuffleResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.response; 20 | 21 | public class RssRegisterShuffleResponse extends ClientResponse { 22 | 23 | public RssRegisterShuffleResponse(ResponseStatusCode statusCode) { 24 | super(statusCode); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/response/RssReportShuffleResultResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.response; 20 | 21 | public class RssReportShuffleResultResponse extends ClientResponse { 22 | 23 | public RssReportShuffleResultResponse(ResponseStatusCode statusCode) { 24 | super(statusCode); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/response/RssSendCommitResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.response; 20 | 21 | public class RssSendCommitResponse extends ClientResponse { 22 | 23 | private int commitCount; 24 | 25 | public RssSendCommitResponse(ResponseStatusCode statusCode) { 26 | super(statusCode); 27 | } 28 | 29 | public int getCommitCount() { 30 | return commitCount; 31 | } 32 | 33 | public void setCommitCount(int commitCount) { 34 | this.commitCount = commitCount; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/response/RssSendHeartBeatResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.response; 20 | 21 | import java.util.Set; 22 | 23 | public class RssSendHeartBeatResponse extends ClientResponse { 24 | 25 | private Set appIds; 26 | 27 | public RssSendHeartBeatResponse(ResponseStatusCode statusCode) { 28 | super(statusCode); 29 | } 30 | 31 | public Set getAppIds() { 32 | return appIds; 33 | } 34 | 35 | public void setAppIds(Set appIds) { 36 | this.appIds = appIds; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/response/RssSendShuffleDataResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.response; 20 | 21 | import java.util.List; 22 | 23 | public class RssSendShuffleDataResponse extends ClientResponse { 24 | 25 | private List successBlockIds; 26 | private List failedBlockIds; 27 | 28 | public RssSendShuffleDataResponse(ResponseStatusCode statusCode) { 29 | super(statusCode); 30 | } 31 | 32 | public List getSuccessBlockIds() { 33 | return successBlockIds; 34 | } 35 | 36 | public void setSuccessBlockIds(List successBlockIds) { 37 | this.successBlockIds = successBlockIds; 38 | } 39 | 40 | public List getFailedBlockIds() { 41 | return failedBlockIds; 42 | } 43 | 44 | public void setFailedBlockIds(List failedBlockIds) { 45 | this.failedBlockIds = failedBlockIds; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /internal-client/src/main/java/com/tencent/rss/client/util/ClientType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.client.util; 20 | 21 | public enum ClientType { 22 | GRPC 23 | } 24 | -------------------------------------------------------------------------------- /server/src/main/java/com/tencent/rss/server/Checker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.server; 20 | 21 | public abstract class Checker { 22 | 23 | protected ShuffleServerConf conf; 24 | 25 | Checker(ShuffleServerConf conf) { 26 | this.conf = conf; 27 | } 28 | 29 | abstract boolean checkIsHealthy(); 30 | } 31 | -------------------------------------------------------------------------------- /server/src/main/java/com/tencent/rss/server/ShuffleDataReadEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.server; 20 | 21 | public class ShuffleDataReadEvent { 22 | 23 | private String appId; 24 | private int shuffleId; 25 | private int startPartition; 26 | 27 | public ShuffleDataReadEvent(String appId, int shuffleId, int startPartition) { 28 | this.appId = appId; 29 | this.shuffleId = shuffleId; 30 | this.startPartition = startPartition; 31 | } 32 | 33 | public String getAppId() { 34 | return appId; 35 | } 36 | 37 | public int getShuffleId() { 38 | return shuffleId; 39 | } 40 | 41 | public int getStartPartition() { 42 | return startPartition; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /server/src/main/java/com/tencent/rss/server/ShuffleServerFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.server; 20 | 21 | import com.tencent.rss.common.rpc.GrpcServer; 22 | import com.tencent.rss.common.rpc.ServerInterface; 23 | 24 | public class ShuffleServerFactory { 25 | 26 | private final ShuffleServer shuffleServer; 27 | private final ShuffleServerConf conf; 28 | 29 | public ShuffleServerFactory(ShuffleServer shuffleServer) { 30 | this.shuffleServer = shuffleServer; 31 | this.conf = shuffleServer.getShuffleServerConf(); 32 | } 33 | 34 | public ServerInterface getServer() { 35 | String type = conf.getString(ShuffleServerConf.RPC_SERVER_TYPE); 36 | if (type.equals(ServerType.GRPC.name())) { 37 | return new GrpcServer(conf, new ShuffleServerGrpcService(shuffleServer), 38 | shuffleServer.getGrpcMetrics()); 39 | } else { 40 | throw new UnsupportedOperationException("Unsupported server type " + type); 41 | } 42 | } 43 | 44 | public ShuffleServer getShuffleServer() { 45 | return shuffleServer; 46 | } 47 | 48 | public ShuffleServerConf getConf() { 49 | return conf; 50 | } 51 | 52 | enum ServerType { 53 | GRPC 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /server/src/main/java/com/tencent/rss/server/StatusCode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.server; 20 | 21 | public enum StatusCode { 22 | SUCCESS(0), 23 | DOUBLE_REGISTER(1), 24 | NO_BUFFER(2), 25 | INVALID_STORAGE(3), 26 | NO_REGISTER(4), 27 | NO_PARTITION(5), 28 | INTERNAL_ERROR(6), 29 | TIMEOUT(7); 30 | 31 | private final int statusCode; 32 | 33 | StatusCode(int code) { 34 | this.statusCode = code; 35 | } 36 | 37 | public int statusCode() { 38 | return statusCode; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /server/src/main/java/com/tencent/rss/server/buffer/PreAllocatedBufferInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.server.buffer; 20 | 21 | public class PreAllocatedBufferInfo { 22 | 23 | private long requireId; 24 | private long timestamp; 25 | private int requireSize; 26 | 27 | public PreAllocatedBufferInfo(long requireId, long timestamp, int requireSize) { 28 | this.requireId = requireId; 29 | this.timestamp = timestamp; 30 | this.requireSize = requireSize; 31 | } 32 | 33 | public long getRequireId() { 34 | return requireId; 35 | } 36 | 37 | public long getTimestamp() { 38 | return timestamp; 39 | } 40 | 41 | public int getRequireSize() { 42 | return requireSize; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /server/src/main/java/com/tencent/rss/server/storage/StorageManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.server.storage; 20 | 21 | import java.util.Set; 22 | 23 | import com.tencent.rss.common.RemoteStorageInfo; 24 | import com.tencent.rss.server.Checker; 25 | import com.tencent.rss.server.ShuffleDataFlushEvent; 26 | import com.tencent.rss.server.ShuffleDataReadEvent; 27 | import com.tencent.rss.storage.common.Storage; 28 | import com.tencent.rss.storage.handler.api.ShuffleWriteHandler; 29 | 30 | 31 | public interface StorageManager { 32 | 33 | Storage selectStorage(ShuffleDataFlushEvent event); 34 | 35 | Storage selectStorage(ShuffleDataReadEvent event); 36 | 37 | boolean write(Storage storage, ShuffleWriteHandler handler, ShuffleDataFlushEvent event); 38 | 39 | void updateWriteMetrics(ShuffleDataFlushEvent event, long writeTime); 40 | 41 | // todo: add an interface for updateReadMetrics 42 | 43 | void removeResources(String appId, Set shuffleSet); 44 | 45 | void start(); 46 | 47 | void stop(); 48 | 49 | void registerRemoteStorage(String appId, RemoteStorageInfo remoteStorageInfo); 50 | 51 | Checker getStorageChecker(); 52 | 53 | // todo: add an interface that check storage isHealthy 54 | } 55 | -------------------------------------------------------------------------------- /server/src/main/java/com/tencent/rss/server/storage/StorageManagerFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.server.storage; 20 | 21 | import com.tencent.rss.server.ShuffleServerConf; 22 | import com.tencent.rss.storage.util.StorageType; 23 | 24 | public class StorageManagerFactory { 25 | 26 | private static class LazyHolder { 27 | static final StorageManagerFactory INSTANCE = new StorageManagerFactory(); 28 | } 29 | 30 | public static StorageManagerFactory getInstance() { 31 | return LazyHolder.INSTANCE; 32 | } 33 | 34 | public StorageManager createStorageManager(String serverId, ShuffleServerConf conf) { 35 | StorageType type = StorageType.valueOf(conf.get(ShuffleServerConf.RSS_STORAGE_TYPE)); 36 | if (StorageType.LOCALFILE.equals(type) || StorageType.MEMORY_LOCALFILE.equals(type)) { 37 | return new LocalStorageManager(conf); 38 | } else if (StorageType.HDFS.equals(type) || StorageType.MEMORY_HDFS.equals(type)) { 39 | return new HdfsStorageManager(conf); 40 | } else if (StorageType.LOCALFILE_HDFS.equals(type) 41 | || StorageType.LOCALFILE_HDFS_2.equals(type) 42 | || StorageType.MEMORY_LOCALFILE_HDFS.equals(type)) { 43 | return new MultiStorageManager(conf, serverId); 44 | } else { 45 | throw new IllegalArgumentException("unknown storageType was found"); 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /server/src/test/java/com/tencent/rss/server/HealthyMockChecker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.server; 20 | 21 | class HealthyMockChecker extends Checker { 22 | 23 | public HealthyMockChecker(ShuffleServerConf conf) { 24 | super(conf); 25 | } 26 | 27 | @Override 28 | boolean checkIsHealthy() { 29 | return true; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /server/src/test/java/com/tencent/rss/server/MockedGrpcServer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.server; 20 | 21 | import com.tencent.rss.common.config.RssBaseConf; 22 | import com.tencent.rss.common.metrics.GRPCMetrics; 23 | import com.tencent.rss.common.rpc.GrpcServer; 24 | 25 | public class MockedGrpcServer extends GrpcServer { 26 | MockedShuffleServerGrpcService service; 27 | public MockedGrpcServer(RssBaseConf conf, MockedShuffleServerGrpcService service, 28 | GRPCMetrics grpcMetrics) { 29 | super(conf, service, grpcMetrics); 30 | this.service = service; 31 | } 32 | public MockedShuffleServerGrpcService getService() { 33 | return service; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /server/src/test/java/com/tencent/rss/server/MockedShuffleServer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.server; 20 | 21 | public class MockedShuffleServer extends ShuffleServer { 22 | public MockedShuffleServer(ShuffleServerConf shuffleServerConf) throws Exception { 23 | super(shuffleServerConf); 24 | } 25 | 26 | @Override 27 | public void setServer() { 28 | setServer(new MockedShuffleServerFactory(this).getServer()); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /server/src/test/java/com/tencent/rss/server/MockedShuffleServerFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.server; 20 | 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | import com.tencent.rss.common.rpc.ServerInterface; 25 | 26 | public class MockedShuffleServerFactory extends ShuffleServerFactory { 27 | private static final Logger LOG = LoggerFactory.getLogger(MockedShuffleServerFactory.class); 28 | public MockedShuffleServerFactory(MockedShuffleServer shuffleServer) { 29 | super(shuffleServer); 30 | } 31 | 32 | @Override 33 | public ServerInterface getServer() { 34 | ShuffleServerConf conf = getConf(); 35 | ShuffleServer shuffleServer = getShuffleServer(); 36 | String type = conf.getString(ShuffleServerConf.RPC_SERVER_TYPE); 37 | if (type.equals(ServerType.GRPC.name())) { 38 | return new MockedGrpcServer(conf, new MockedShuffleServerGrpcService(shuffleServer), 39 | shuffleServer.getGrpcMetrics()); 40 | } else { 41 | throw new UnsupportedOperationException("Unsupported server type " + type); 42 | } 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /server/src/test/java/com/tencent/rss/server/UnHealthyMockChecker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.server; 20 | 21 | class UnHealthyMockChecker extends Checker { 22 | 23 | public UnHealthyMockChecker(ShuffleServerConf conf) { 24 | super(conf); 25 | } 26 | 27 | @Override 28 | boolean checkIsHealthy() { 29 | return false; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /server/src/test/java/com/tencent/rss/server/buffer/BufferTestBase.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.server.buffer; 20 | 21 | import java.util.Random; 22 | import java.util.concurrent.atomic.AtomicLong; 23 | 24 | import org.junit.jupiter.api.AfterAll; 25 | import org.junit.jupiter.api.BeforeAll; 26 | 27 | import com.tencent.rss.common.ShufflePartitionedBlock; 28 | import com.tencent.rss.common.ShufflePartitionedData; 29 | import com.tencent.rss.common.util.ChecksumUtils; 30 | import com.tencent.rss.server.ShuffleServerMetrics; 31 | 32 | public abstract class BufferTestBase { 33 | 34 | @BeforeAll 35 | public static void setup() { 36 | ShuffleServerMetrics.register(); 37 | } 38 | 39 | @AfterAll 40 | public static void clear() { 41 | ShuffleServerMetrics.clear(); 42 | } 43 | 44 | private static AtomicLong atomBlockId = new AtomicLong(0); 45 | 46 | protected ShufflePartitionedData createData(int len) { 47 | return createData(1, len); 48 | } 49 | 50 | protected ShufflePartitionedData createData(int partitionId, int len) { 51 | byte[] buf = new byte[len]; 52 | new Random().nextBytes(buf); 53 | ShufflePartitionedBlock block = new ShufflePartitionedBlock( 54 | len, len, ChecksumUtils.getCrc32(buf), atomBlockId.incrementAndGet(), 0, buf); 55 | ShufflePartitionedData data = new ShufflePartitionedData( 56 | partitionId, new ShufflePartitionedBlock[]{block}); 57 | return data; 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /server/src/test/resources/confBySizeStringTest.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Tencent is pleased to support the open source community by making 3 | # Firestorm-Spark remote shuffle server available. 4 | # 5 | # Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | # this file except in compliance with the License. You may obtain a copy of the 9 | # License at 10 | # 11 | # https://opensource.org/licenses/Apache-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | # WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations under the License. 17 | # 18 | 19 | 20 | rss.server.buffer.capacity 10240 21 | rss.server.buffer.spill.threshold 1gb 22 | rss.server.partition.buffer.size 2MB 23 | rss.server.read.buffer.capacity 32kb 24 | rss.server.write.slow.threshold 10gb 25 | rss.server.event.size.threshold.l1 45mb 26 | rss.server.event.size.threshold.l2 90mb 27 | rss.server.event.size.threshold.l3 120mb 28 | rss.server.disk.capacity 120gb 29 | rss.server.shuffle.max.upload.size 45mb 30 | -------------------------------------------------------------------------------- /server/src/test/resources/confTest.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Tencent is pleased to support the open source community by making 3 | # Firestorm-Spark remote shuffle server available. 4 | # 5 | # Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | # this file except in compliance with the License. You may obtain a copy of the 9 | # License at 10 | # 11 | # https://opensource.org/licenses/Apache-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | # WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations under the License. 17 | # 18 | 19 | rss.rpc.server.port 1234 20 | rss.storage.type FILE 21 | rss.storage.basePath /var/tmp/test 22 | rss.server.coordinator.ip localhost 23 | rss.server.coordinator.port 9527 24 | rss.server.buffer.capacity 2 25 | rss.server.hadoop.a.b value1 26 | rss.server.had.a.b value2 27 | rss.server.uploader.remote.sTorage.type COS 28 | rss.server.multistorage.enable true 29 | rss.server.uploader.enable true 30 | rss.server.uploader.thread.number 13 31 | rss.server.cluster.hadoop.clustere1. 32 | -------------------------------------------------------------------------------- /server/src/test/resources/gcTest.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Tencent is pleased to support the open source community by making 3 | # Firestorm-Spark remote shuffle server available. 4 | # 5 | # Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | # this file except in compliance with the License. You may obtain a copy of the 9 | # License at 10 | # 11 | # https://opensource.org/licenses/Apache-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | # WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations under the License. 17 | # 18 | 19 | rss.rpc.server.port 1234 20 | rss.server.coordinator.ip localhost 21 | rss.server.coordinator.port 9527 22 | rss.server.gc.threshold 3 23 | rss.server.buffer.capacity 12 24 | rss.server.partition.buffer.size 128 25 | rss.jetty.http.port 12345 26 | rss.jetty.corePool.size 64 27 | rss.storage.basePath /fake/path -------------------------------------------------------------------------------- /server/src/test/resources/server.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Tencent is pleased to support the open source community by making 3 | # Firestorm-Spark remote shuffle server available. 4 | # 5 | # Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | # this file except in compliance with the License. You may obtain a copy of the 9 | # License at 10 | # 11 | # https://opensource.org/licenses/Apache-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | # WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations under the License. 17 | # 18 | 19 | rss.rpc.server.port 1234 20 | rss.storage.type HDFS 21 | rss.storage.basePath /var/tmp/test 22 | rss.coordinator.quorum localhost:9527 23 | rss.server.buffer.capacity 150 24 | rss.server.buffer.spill.threshold 130 25 | rss.server.partition.buffer.size 128 26 | rss.jetty.http.port 12345 27 | rss.jetty.corePool.size 64 28 | rss.server.heartbeat.timeout 1 29 | rss.server.write.timeout 2000 -------------------------------------------------------------------------------- /spotbugs-exclude.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /storage/src/main/java/com/tencent/rss/storage/api/FileReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.storage.api; 20 | 21 | public interface FileReader { 22 | 23 | byte[] read(long offset, int length); 24 | 25 | byte[] read(); 26 | } 27 | -------------------------------------------------------------------------------- /storage/src/main/java/com/tencent/rss/storage/common/ShuffleInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.storage.common; 20 | 21 | public class ShuffleInfo { 22 | private String key; 23 | private long size; 24 | 25 | public ShuffleInfo(String key, long size) { 26 | this.key = key; 27 | this.size = size; 28 | } 29 | 30 | public String getKey() { 31 | return key; 32 | } 33 | 34 | public long getSize() { 35 | return size; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /storage/src/main/java/com/tencent/rss/storage/common/ShuffleSegment.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.storage.common; 20 | 21 | public abstract class ShuffleSegment implements java.io.Serializable { 22 | 23 | } 24 | -------------------------------------------------------------------------------- /storage/src/main/java/com/tencent/rss/storage/common/Storage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.storage.common; 20 | 21 | import java.io.IOException; 22 | 23 | import com.tencent.rss.storage.handler.api.ServerReadHandler; 24 | import com.tencent.rss.storage.handler.api.ShuffleWriteHandler; 25 | import com.tencent.rss.storage.request.CreateShuffleReadHandlerRequest; 26 | import com.tencent.rss.storage.request.CreateShuffleWriteHandlerRequest; 27 | 28 | 29 | public interface Storage { 30 | 31 | boolean canWrite(); 32 | 33 | boolean lockShuffleShared(String shuffleKey); 34 | 35 | boolean unlockShuffleShared(String shuffleKey); 36 | 37 | boolean lockShuffleExcluded(String shuffleKey); 38 | 39 | boolean unlockShuffleExcluded(String shuffleKey); 40 | 41 | void updateWriteMetrics(StorageWriteMetrics metrics); 42 | 43 | void updateReadMetrics(StorageReadMetrics metrics); 44 | 45 | ShuffleWriteHandler getOrCreateWriteHandler(CreateShuffleWriteHandlerRequest request) throws IOException; 46 | 47 | ServerReadHandler getOrCreateReadHandler(CreateShuffleReadHandlerRequest request); 48 | 49 | CreateShuffleWriteHandlerRequest getCreateWriterHandlerRequest(String appId, int shuffleId, int partition); 50 | 51 | void removeHandlers(String appId); 52 | 53 | void createMetadataIfNotExist(String shuffleKey); 54 | 55 | String getStoragePath(); 56 | 57 | String getStorageHost(); 58 | } 59 | -------------------------------------------------------------------------------- /storage/src/main/java/com/tencent/rss/storage/common/StorageReadMetrics.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.storage.common; 20 | 21 | public class StorageReadMetrics { 22 | 23 | private long lastReadTs; 24 | private String appId; 25 | private int shuffleId; 26 | 27 | public StorageReadMetrics(String appId, int shuffleId) { 28 | this.appId = appId; 29 | this.shuffleId = shuffleId; 30 | lastReadTs = System.currentTimeMillis(); 31 | } 32 | 33 | public long getLastReadTs() { 34 | return lastReadTs; 35 | } 36 | 37 | public String getAppId() { 38 | return appId; 39 | } 40 | 41 | public int getShuffleId() { 42 | return shuffleId; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /storage/src/main/java/com/tencent/rss/storage/common/StorageWriteMetrics.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.storage.common; 20 | 21 | import java.util.List; 22 | 23 | public class StorageWriteMetrics { 24 | 25 | private final String appId; 26 | private final int shuffleId; 27 | private final long eventSize; 28 | private final long writeBlocks; 29 | private final long writeTime; 30 | private final long dataSize; 31 | private final List partitions; 32 | 33 | public StorageWriteMetrics( 34 | long eventSize, 35 | long writeBlocks, 36 | long writeTime, 37 | long dataSize, 38 | List partitions, 39 | String appId, 40 | int shuffleId) { 41 | this.writeBlocks = writeBlocks; 42 | this.eventSize = eventSize; 43 | this.writeTime = writeTime; 44 | this.dataSize = dataSize; 45 | this.partitions = partitions; 46 | this.appId = appId; 47 | this.shuffleId = shuffleId; 48 | } 49 | 50 | public long getEventSize() { 51 | return eventSize; 52 | } 53 | 54 | public long getWriteBlocks() { 55 | return writeBlocks; 56 | } 57 | 58 | public long getWriteTime() { 59 | return writeTime; 60 | } 61 | 62 | public long getDataSize() { 63 | return dataSize; 64 | } 65 | 66 | public List getPartitions() { 67 | return partitions; 68 | } 69 | 70 | public String getAppId() { 71 | return appId; 72 | } 73 | 74 | public int getShuffleId() { 75 | return shuffleId; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /storage/src/main/java/com/tencent/rss/storage/factory/ShuffleUploadHandlerFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.storage.factory; 20 | 21 | import java.io.IOException; 22 | 23 | import com.tencent.rss.storage.handler.api.ShuffleUploadHandler; 24 | import com.tencent.rss.storage.handler.impl.HdfsShuffleUploadHandler; 25 | import com.tencent.rss.storage.request.CreateShuffleUploadHandlerRequest; 26 | import com.tencent.rss.storage.util.StorageType; 27 | 28 | public class ShuffleUploadHandlerFactory { 29 | 30 | public ShuffleUploadHandlerFactory() { 31 | 32 | } 33 | 34 | private static class LazyHolder { 35 | static final ShuffleUploadHandlerFactory INSTANCE = new ShuffleUploadHandlerFactory(); 36 | } 37 | 38 | public static ShuffleUploadHandlerFactory getInstance() { 39 | return LazyHolder.INSTANCE; 40 | } 41 | 42 | public ShuffleUploadHandler createShuffleUploadHandler( 43 | CreateShuffleUploadHandlerRequest request) throws IOException, RuntimeException { 44 | if (request.getRemoteStorageType() == StorageType.HDFS) { 45 | return new HdfsShuffleUploadHandler( 46 | request.getRemoteStorageBasePath(), 47 | request.getHadoopConf(), 48 | request.getHdfsFilePrefix(), 49 | request.getBufferSize(), 50 | request.getCombineUpload()); 51 | } else { 52 | throw new RuntimeException("Unsupported remote storage type " + request.getRemoteStorageType().name()); 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /storage/src/main/java/com/tencent/rss/storage/handler/api/ClientReadHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.storage.handler.api; 20 | 21 | import com.tencent.rss.common.BufferSegment; 22 | import com.tencent.rss.common.ShuffleDataResult; 23 | 24 | public interface ClientReadHandler { 25 | 26 | ShuffleDataResult readShuffleData(); 27 | 28 | void close(); 29 | 30 | // The handler only returns the segment, 31 | // but does not know the actually consumed blocks, 32 | // so the consumer should let the handler update statistics. 33 | // Each type of handler can design their rules. 34 | void updateConsumedBlockInfo(BufferSegment bs); 35 | 36 | // Display the statistics of consumed blocks 37 | void logConsumedBlockInfo(); 38 | 39 | } 40 | -------------------------------------------------------------------------------- /storage/src/main/java/com/tencent/rss/storage/handler/api/ServerReadHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.storage.handler.api; 20 | 21 | import com.tencent.rss.common.ShuffleDataResult; 22 | import com.tencent.rss.common.ShuffleIndexResult; 23 | 24 | public interface ServerReadHandler { 25 | 26 | ShuffleDataResult getShuffleData(long offset, int length); 27 | 28 | ShuffleIndexResult getShuffleIndex(); 29 | 30 | } 31 | -------------------------------------------------------------------------------- /storage/src/main/java/com/tencent/rss/storage/handler/api/ShuffleDeleteHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.storage.handler.api; 20 | 21 | public interface ShuffleDeleteHandler { 22 | 23 | /** 24 | * Delete shuffle data with appId 25 | * 26 | * @param appId ApplicationId for delete 27 | */ 28 | void delete(String[] storageBasePaths, String appId); 29 | } 30 | -------------------------------------------------------------------------------- /storage/src/main/java/com/tencent/rss/storage/handler/api/ShuffleUploadHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.storage.handler.api; 20 | 21 | import java.io.File; 22 | import java.util.List; 23 | 24 | import com.tencent.rss.storage.util.ShuffleUploadResult; 25 | 26 | public interface ShuffleUploadHandler { 27 | 28 | /** 29 | * Upload data files and index files to remote storage, the correctness of relation between 30 | * items of data files, index files and partitions is guaranteed by caller/user. 31 | * Normally we best-effort strategy to upload files, so the result may be part of the files. 32 | * 33 | * @param dataFiles local data files to be uploaded to remote storage 34 | * @param indexFiles local index files to be uploaded to remote storage 35 | * @param partitions partition id of the local data files and index files 36 | * 37 | * @return upload result including total successful uploaded file size and partition id list 38 | * 39 | */ 40 | ShuffleUploadResult upload(List dataFiles, List indexFiles, List partitions); 41 | 42 | } 43 | -------------------------------------------------------------------------------- /storage/src/main/java/com/tencent/rss/storage/handler/api/ShuffleWriteHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.storage.handler.api; 20 | 21 | import java.io.IOException; 22 | import java.util.List; 23 | 24 | import com.tencent.rss.common.ShufflePartitionedBlock; 25 | 26 | public interface ShuffleWriteHandler { 27 | 28 | /** 29 | * Write the blocks to storage 30 | * 31 | * @param shuffleBlocks blocks to storage 32 | * @throws IOException 33 | * @throws IllegalStateException 34 | */ 35 | void write(List shuffleBlocks) throws IOException, IllegalStateException; 36 | } 37 | -------------------------------------------------------------------------------- /storage/src/main/java/com/tencent/rss/storage/handler/impl/AbstractClientReadHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.storage.handler.impl; 20 | 21 | import com.tencent.rss.common.BufferSegment; 22 | import com.tencent.rss.common.ShuffleDataResult; 23 | import com.tencent.rss.storage.handler.api.ClientReadHandler; 24 | 25 | public abstract class AbstractClientReadHandler implements ClientReadHandler { 26 | 27 | protected String appId; 28 | protected int shuffleId; 29 | protected int partitionId; 30 | protected int readBufferSize; 31 | 32 | @Override 33 | public ShuffleDataResult readShuffleData() { 34 | return null; 35 | } 36 | 37 | @Override 38 | public void close() { 39 | } 40 | 41 | @Override 42 | public void updateConsumedBlockInfo(BufferSegment bs) { 43 | } 44 | 45 | @Override 46 | public void logConsumedBlockInfo() { 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /storage/src/main/java/com/tencent/rss/storage/handler/impl/DataFileSegment.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.storage.handler.impl; 20 | 21 | import java.util.List; 22 | import java.util.Set; 23 | 24 | import com.google.common.collect.Sets; 25 | 26 | import com.tencent.rss.common.BufferSegment; 27 | 28 | public class DataFileSegment extends FileSegment { 29 | 30 | private List bufferSegments; 31 | 32 | public DataFileSegment(String path, long offset, int length, List bufferSegments) { 33 | super(path, offset, length); 34 | this.bufferSegments = bufferSegments; 35 | } 36 | 37 | public List getBufferSegments() { 38 | return bufferSegments; 39 | } 40 | 41 | public Set getBlockIds() { 42 | Set blockIds = Sets.newHashSet(); 43 | for (BufferSegment bs : bufferSegments) { 44 | blockIds.add(bs.getBlockId()); 45 | } 46 | return blockIds; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /storage/src/main/java/com/tencent/rss/storage/handler/impl/FileSegment.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.storage.handler.impl; 20 | 21 | public class FileSegment { 22 | 23 | private String path; 24 | private long offset; 25 | private int length; 26 | 27 | public FileSegment(String path, long offset, int length) { 28 | this.path = path; 29 | this.offset = offset; 30 | this.length = length; 31 | } 32 | 33 | public String getPath() { 34 | return path; 35 | } 36 | 37 | public long getOffset() { 38 | return offset; 39 | } 40 | 41 | public int getLength() { 42 | return length; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /storage/src/main/java/com/tencent/rss/storage/handler/impl/LocalFileDeleteHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.storage.handler.impl; 20 | 21 | import java.io.File; 22 | 23 | import org.apache.commons.io.FileUtils; 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | 27 | import com.tencent.rss.storage.handler.api.ShuffleDeleteHandler; 28 | import com.tencent.rss.storage.util.ShuffleStorageUtils; 29 | 30 | public class LocalFileDeleteHandler implements ShuffleDeleteHandler { 31 | 32 | private static final Logger LOG = LoggerFactory.getLogger(LocalFileDeleteHandler.class); 33 | 34 | @Override 35 | public void delete(String[] storageBasePaths, String appId) { 36 | for (String basePath : storageBasePaths) { 37 | String shufflePath = ShuffleStorageUtils.getFullShuffleDataFolder(basePath, appId); 38 | long start = System.currentTimeMillis(); 39 | try { 40 | File baseFolder = new File(shufflePath); 41 | FileUtils.deleteDirectory(baseFolder); 42 | LOG.info("Delete shuffle data for appId[" + appId + "] with " + shufflePath 43 | + " cost " + (System.currentTimeMillis() - start) + " ms"); 44 | } catch (Exception e) { 45 | LOG.warn("Can't delete shuffle data for appId[" + appId + "] with " + shufflePath, e); 46 | } 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /storage/src/main/java/com/tencent/rss/storage/request/CreateShuffleDeleteHandlerRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.storage.request; 20 | 21 | import org.apache.hadoop.conf.Configuration; 22 | 23 | public class CreateShuffleDeleteHandlerRequest { 24 | 25 | private String storageType; 26 | private Configuration conf; 27 | 28 | public CreateShuffleDeleteHandlerRequest(String storageType, Configuration conf) { 29 | this.storageType = storageType; 30 | this.conf = conf; 31 | } 32 | 33 | public String getStorageType() { 34 | return storageType; 35 | } 36 | 37 | public Configuration getConf() { 38 | return conf; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /storage/src/main/java/com/tencent/rss/storage/util/StorageType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.storage.util; 20 | 21 | public enum StorageType { 22 | HDFS, 23 | LOCALFILE, 24 | LOCALFILE_HDFS, 25 | LOCALFILE_HDFS_2, 26 | MEMORY_LOCALFILE, 27 | MEMORY_HDFS, 28 | MEMORY_LOCALFILE_HDFS 29 | } 30 | -------------------------------------------------------------------------------- /storage/src/test/java/com/tencent/rss/storage/common/ShuffleFileInfoTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making 3 | * Firestorm-Spark remote shuffle server available. 4 | * 5 | * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package com.tencent.rss.storage.common; 20 | 21 | import java.io.File; 22 | import org.junit.jupiter.api.Test; 23 | 24 | import static org.junit.jupiter.api.Assertions.assertFalse; 25 | import static org.junit.jupiter.api.Assertions.assertTrue; 26 | import static org.junit.jupiter.api.Assertions.fail; 27 | 28 | public class ShuffleFileInfoTest { 29 | 30 | @Test 31 | public void test() { 32 | try { 33 | ShuffleFileInfo shuffleFileInfo = new ShuffleFileInfo(); 34 | shuffleFileInfo.getDataFiles().add(File.createTempFile("dummy-data-file", ".data")); 35 | shuffleFileInfo.setKey("key"); 36 | assertFalse(shuffleFileInfo.isValid()); 37 | 38 | shuffleFileInfo.getIndexFiles().add(File.createTempFile("dummy-data-file", ".index")); 39 | shuffleFileInfo.getPartitions().add(12); 40 | shuffleFileInfo.setSize(1024 * 1024 * 32); 41 | assertTrue(shuffleFileInfo.isValid()); 42 | assertFalse(shuffleFileInfo.shouldCombine(32)); 43 | } catch (Exception e) { 44 | e.printStackTrace(); 45 | fail(e.getMessage()); 46 | } 47 | } 48 | } 49 | --------------------------------------------------------------------------------