├── connectors ├── json │ ├── src │ │ ├── test │ │ │ └── resources │ │ │ │ ├── empty.json │ │ │ │ ├── multi_doc.json.z │ │ │ │ ├── multi_doc.json.br │ │ │ │ ├── multi_doc.json.bz2 │ │ │ │ ├── multi_doc.json.gz │ │ │ │ ├── multi_doc.json.lz4 │ │ │ │ ├── multi_doc.json.xz │ │ │ │ ├── multi_doc.json.lzma │ │ │ │ ├── multi_doc.json.snappy │ │ │ │ ├── multi_doc.json.zstd │ │ │ │ ├── multi_doc.json │ │ │ │ ├── single_doc.json │ │ │ │ └── logback-test.xml │ │ └── main │ │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ └── com.datastax.oss.dsbulk.connectors.api.Connector │ └── README.md ├── csv │ ├── src │ │ ├── test │ │ │ └── resources │ │ │ │ ├── bad_header_empty.csv │ │ │ │ ├── bad_header_duplicate.csv │ │ │ │ ├── multi-char-delimiter.csv │ │ │ │ ├── sample.csv.z │ │ │ │ ├── sample.csv.br │ │ │ │ ├── sample.csv.bz2 │ │ │ │ ├── sample.csv.gz │ │ │ │ ├── sample.csv.lz4 │ │ │ │ ├── sample.csv.xz │ │ │ │ ├── sample.csv.lzma │ │ │ │ ├── sample.csv.snappy │ │ │ │ ├── sample.csv.zstd │ │ │ │ ├── sample.csv │ │ │ │ └── logback-test.xml │ │ └── main │ │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ └── com.datastax.oss.dsbulk.connectors.api.Connector │ └── README.md ├── api │ ├── README.md │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── datastax │ │ └── oss │ │ └── dsbulk │ │ └── connectors │ │ └── api │ │ ├── ConnectorFeature.java │ │ ├── IndexedField.java │ │ ├── MappedField.java │ │ ├── Field.java │ │ ├── Resource.java │ │ ├── DefaultResource.java │ │ └── RecordMetadata.java ├── commons │ └── README.md ├── README.md └── pom.xml ├── runner ├── src │ ├── it │ │ └── resources │ │ │ ├── counters.csv │ │ │ ├── duplicates.csv │ │ │ ├── bad_header_empty.csv │ │ │ ├── function-pk.csv │ │ │ ├── bad_header_duplicate.csv │ │ │ ├── custom-type.csv │ │ │ ├── invalid-mapping.csv │ │ │ ├── with-spaces.csv │ │ │ ├── missing-extra.csv │ │ │ ├── empty-strings.json │ │ │ ├── numeric-fields.json │ │ │ ├── empty-blobs.json │ │ │ ├── temporal-numeric.csv │ │ │ ├── with-spaces.json │ │ │ ├── comments.json │ │ │ ├── META-INF │ │ │ └── services │ │ │ │ └── com.datastax.oss.dsbulk.connectors.api.Connector │ │ │ ├── ttl-timestamp.csv │ │ │ ├── ip-by-country-unique.csv.gz │ │ │ ├── temporal.csv │ │ │ ├── temporal.json │ │ │ ├── ttl-timestamp-unset.csv │ │ │ ├── missing-extra.json │ │ │ ├── numbers-missing.json │ │ │ ├── missing.json │ │ │ ├── complex.csv │ │ │ ├── missing-case.json │ │ │ ├── metrics │ │ │ └── prometheus.yml │ │ │ ├── ip-by-country-unique-part-1.csv │ │ │ ├── part_1_csv │ │ │ └── ip-by-country-unique-part-1.csv │ │ │ ├── application.conf │ │ │ ├── complex.json │ │ │ ├── ip-by-country-unique-part-2.csv │ │ │ ├── part_2_csv │ │ │ └── ip-by-country-unique-part-2.csv │ │ │ ├── graph │ │ │ └── customer-orders.csv │ │ │ └── number.csv │ ├── test │ │ └── resources │ │ │ └── bad-json.conf │ └── main │ │ └── java │ │ └── com │ │ └── datastax │ │ └── oss │ │ └── dsbulk │ │ └── runner │ │ ├── cli │ │ ├── VersionRequestException.java │ │ ├── ParseException.java │ │ ├── GlobalHelpRequestException.java │ │ ├── ParsedCommandLine.java │ │ └── SectionHelpRequestException.java │ │ └── ExitStatus.java └── README.md ├── .gitattributes ├── io ├── src │ └── test │ │ ├── resources │ │ ├── compression │ │ │ ├── test-file │ │ │ ├── test.br │ │ │ ├── test.gz │ │ │ ├── test.xz │ │ │ ├── test.z │ │ │ ├── test.bz2 │ │ │ ├── test.lz4 │ │ │ ├── test.lzma │ │ │ ├── test.zstd │ │ │ └── test.snappy │ │ └── logback-test.xml │ │ └── java │ │ └── com │ │ └── datastax │ │ └── oss │ │ └── dsbulk │ │ └── io │ │ └── IOUtilsTest.java └── README.md ├── workflow ├── api │ ├── src │ │ └── main │ │ │ ├── resources │ │ │ └── com │ │ │ │ └── datastax │ │ │ │ └── oss │ │ │ │ └── dsbulk │ │ │ │ └── version.txt │ │ │ └── java │ │ │ └── com │ │ │ └── datastax │ │ │ └── oss │ │ │ └── dsbulk │ │ │ └── workflow │ │ │ └── api │ │ │ ├── error │ │ │ ├── UnlimitedErrorThreshold.java │ │ │ ├── TooManyErrorsException.java │ │ │ └── AbsoluteErrorThreshold.java │ │ │ ├── utils │ │ │ └── ConsoleUtils.java │ │ │ └── log │ │ │ └── OperationDirectory.java │ └── README.md ├── commons │ ├── src │ │ ├── test │ │ │ └── resources │ │ │ │ ├── application.conf │ │ │ │ ├── mockito-extensions │ │ │ │ └── org.mockito.plugins.MockMaker │ │ │ │ └── logback-test.xml │ │ └── main │ │ │ ├── resources │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── com.datastax.oss.dsbulk.workflow.api.config.ConfigPostProcessor │ │ │ └── java │ │ │ └── com │ │ │ └── datastax │ │ │ └── oss │ │ │ └── dsbulk │ │ │ └── workflow │ │ │ └── commons │ │ │ ├── schema │ │ │ ├── NestedBatchException.java │ │ │ ├── ReadResultMapper.java │ │ │ ├── RecordMapper.java │ │ │ └── ReadResultCounter.java │ │ │ ├── log │ │ │ └── RangeReadResource.java │ │ │ ├── utils │ │ │ └── StringUtils.java │ │ │ ├── statement │ │ │ ├── MappedStatement.java │ │ │ └── UnmappableStatement.java │ │ │ ├── auth │ │ │ └── BulkGssApiAuthProvider.java │ │ │ └── settings │ │ │ └── RowType.java │ └── README.md ├── load │ ├── src │ │ └── main │ │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ └── com.datastax.oss.dsbulk.workflow.api.WorkflowProvider │ └── README.md ├── count │ ├── src │ │ └── main │ │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ └── com.datastax.oss.dsbulk.workflow.api.WorkflowProvider │ └── README.md ├── unload │ ├── src │ │ └── main │ │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ └── com.datastax.oss.dsbulk.workflow.api.WorkflowProvider │ └── README.md ├── README.md └── pom.xml ├── config ├── src │ ├── test │ │ └── resources │ │ │ ├── mockito-extensions │ │ │ └── org.mockito.plugins.MockMaker │ │ │ ├── application.conf │ │ │ ├── application-custom.conf │ │ │ ├── reference.conf │ │ │ ├── dsbulk-reference.conf │ │ │ └── logback-test.xml │ └── main │ │ └── java │ │ └── com │ │ └── datastax │ │ └── oss │ │ └── dsbulk │ │ └── config │ │ └── model │ │ ├── OrderedSettingsGroup.java │ │ ├── SettingsGroup.java │ │ └── FixedSettingsGroup.java └── README.md ├── tests ├── src │ └── main │ │ ├── resources │ │ ├── ssl │ │ │ ├── client.keystore │ │ │ ├── client.truststore │ │ │ ├── server.keystore │ │ │ ├── server.truststore │ │ │ ├── server_localhost.keystore │ │ │ └── client.crt │ │ └── logback-template.xml │ │ └── java │ │ └── com │ │ └── datastax │ │ └── oss │ │ └── dsbulk │ │ └── tests │ │ ├── logging │ │ ├── StreamType.java │ │ ├── StreamCapture.java │ │ ├── LogCapture.java │ │ ├── LogInterceptor.java │ │ └── LogConfigurationResource.java │ │ ├── utils │ │ ├── PlatformUtils.java │ │ └── MemoryUtils.java │ │ ├── ccm │ │ └── annotations │ │ │ ├── CCMRequirements.java │ │ │ ├── CCMFactory.java │ │ │ └── CCMWorkload.java │ │ ├── driver │ │ └── annotations │ │ │ └── SessionFactory.java │ │ └── simulacron │ │ └── annotations │ │ └── SimulacronFactory.java └── README.md ├── batcher ├── reactor │ ├── src │ │ ├── main │ │ │ └── resources │ │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── com.datastax.oss.dsbulk.batcher.api.ReactiveStatementBatcherFactory │ │ └── test │ │ │ └── resources │ │ │ └── logback-test.xml │ └── README.md ├── api │ ├── README.md │ └── src │ │ └── test │ │ └── resources │ │ └── logback-test.xml ├── README.md └── pom.xml ├── executor ├── reactor │ ├── src │ │ ├── main │ │ │ ├── resources │ │ │ │ └── META-INF │ │ │ │ │ └── services │ │ │ │ │ └── com.datastax.oss.dsbulk.executor.api.BulkExecutorBuilderFactory │ │ │ └── java │ │ │ │ └── com │ │ │ │ └── datastax │ │ │ │ └── oss │ │ │ │ └── dsbulk │ │ │ │ └── executor │ │ │ │ └── reactor │ │ │ │ ├── ReactorBulkExecutor.java │ │ │ │ ├── DefaultReactorBulkExecutorBuilder.java │ │ │ │ ├── ContinuousReactorBulkExecutorBuilder.java │ │ │ │ └── ReactorBulkExecutorBuilderFactory.java │ │ └── test │ │ │ ├── resources │ │ │ └── logback-test.xml │ │ │ └── java │ │ │ └── com │ │ │ └── datastax │ │ │ └── oss │ │ │ └── dsbulk │ │ │ └── executor │ │ │ └── reactor │ │ │ └── DefaultReactorBulkExecutorTest.java │ └── README.md ├── api │ ├── README.md │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── datastax │ │ │ └── oss │ │ │ └── dsbulk │ │ │ └── executor │ │ │ └── api │ │ │ ├── SyncBulkExecutor.java │ │ │ ├── AsyncBulkExecutor.java │ │ │ ├── ReactiveBulkExecutor.java │ │ │ ├── BulkExecutor.java │ │ │ └── BulkExecutorBuilderFactory.java │ │ └── test │ │ └── resources │ │ └── logback-test.xml ├── README.md └── pom.xml ├── .gitignore ├── codecs ├── text │ ├── README.md │ └── src │ │ ├── main │ │ ├── resources │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── com.datastax.oss.dsbulk.codecs.api.ConvertingCodecProvider │ │ └── java │ │ │ └── com │ │ │ └── datastax │ │ │ └── oss │ │ │ └── dsbulk │ │ │ └── codecs │ │ │ └── text │ │ │ ├── string │ │ │ ├── StringToListCodec.java │ │ │ ├── StringToSetCodec.java │ │ │ ├── StringToVectorCodec.java │ │ │ ├── StringToStringCodec.java │ │ │ └── StringToDurationCodec.java │ │ │ ├── TextConversionContext.java │ │ │ └── json │ │ │ └── JsonNodeToListCodec.java │ │ └── test │ │ └── resources │ │ └── logback-test.xml ├── jdk │ └── src │ │ ├── main │ │ ├── resources │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── com.datastax.oss.dsbulk.codecs.api.ConvertingCodecProvider │ │ └── java │ │ │ └── com │ │ │ └── datastax │ │ │ └── oss │ │ │ └── dsbulk │ │ │ └── codecs │ │ │ └── jdk │ │ │ └── bool │ │ │ └── BooleanToStringCodec.java │ │ ├── README.md │ │ └── test │ │ ├── resources │ │ └── logback-test.xml │ │ └── java │ │ └── com │ │ └── datastax │ │ └── oss │ │ └── dsbulk │ │ └── codecs │ │ └── jdk │ │ └── bool │ │ └── BooleanToStringCodecTest.java ├── README.md ├── api │ ├── README.md │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── datastax │ │ │ └── oss │ │ │ └── dsbulk │ │ │ └── codecs │ │ │ └── api │ │ │ ├── format │ │ │ ├── geo │ │ │ │ ├── GeoFormatTest.java │ │ │ │ ├── JsonGeoFormat.java │ │ │ │ ├── WellKnownTextGeoFormat.java │ │ │ │ └── GeoFormat.java │ │ │ └── binary │ │ │ │ ├── HexBinaryFormat.java │ │ │ │ └── BinaryFormat.java │ │ │ ├── IdempotentConvertingCodec.java │ │ │ ├── ConversionContext.java │ │ │ └── ConvertingCodecProvider.java │ │ └── test │ │ └── resources │ │ └── logback-test.xml └── pom.xml ├── url ├── src │ ├── main │ │ ├── resources │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── com.datastax.oss.dsbulk.url.URLStreamHandlerProvider │ │ └── java │ │ │ └── com │ │ │ └── datastax │ │ │ └── oss │ │ │ └── dsbulk │ │ │ └── url │ │ │ ├── UncloseableInputStream.java │ │ │ ├── UncloseableOutputStream.java │ │ │ └── StdinStdoutURLStreamHandlerProvider.java │ └── test │ │ ├── resources │ │ └── logback-test.xml │ │ └── java │ │ └── com │ │ └── datastax │ │ └── oss │ │ └── dsbulk │ │ └── url │ │ ├── UncloseableOutputStreamTest.java │ │ ├── UncloseableInputStreamTest.java │ │ ├── StdinStdoutURLStreamHandlerProviderTest.java │ │ └── S3URLStreamHandlerProviderTest.java └── README.md ├── cql └── README.md ├── format ├── README.md └── src │ ├── main │ └── java │ │ └── com │ │ └── datastax │ │ └── oss │ │ └── dsbulk │ │ └── format │ │ ├── statement │ │ └── DefaultStatementPrinter.java │ │ └── row │ │ └── RowFormatterSymbols.java │ └── test │ └── resources │ └── logback-test.xml ├── sampler ├── README.md └── src │ ├── test │ └── resources │ │ └── logback-test.xml │ └── main │ └── java │ └── com │ └── datastax │ └── oss │ └── dsbulk │ └── sampler │ └── Sizeable.java ├── docs └── README.md ├── appveyor.yml ├── uploadtests.ps1 ├── mapping └── src │ ├── main │ └── java │ │ └── com │ │ └── datastax │ │ └── oss │ │ └── dsbulk │ │ └── mapping │ │ ├── MappingToken.java │ │ ├── MappingPreference.java │ │ ├── IndexedMappingField.java │ │ ├── MappingField.java │ │ ├── MappedMappingField.java │ │ └── CQLFragment.java │ └── test │ ├── resources │ └── logback-test.xml │ └── java │ └── com │ └── datastax │ └── oss │ └── dsbulk │ └── mapping │ └── TypedCQLLiteralTest.java ├── partitioner ├── README.md └── src │ ├── test │ ├── resources │ │ └── logback-test.xml │ └── java │ │ └── com │ │ └── datastax │ │ └── oss │ │ └── dsbulk │ │ └── partitioner │ │ └── assertions │ │ └── PartitionerAssertions.java │ ├── main │ └── java │ │ └── com │ │ └── datastax │ │ └── oss │ │ └── dsbulk │ │ └── partitioner │ │ └── BulkTokenRange.java │ └── it │ └── java │ └── com │ └── datastax │ └── oss │ └── dsbulk │ └── partitioner │ └── M3PTokenPartitionerCCMIT.java ├── .travis.yml ├── ci └── settings.xml ├── distribution └── src │ ├── assembly │ └── sources.xml │ └── conf │ └── driver.conf └── appveyor.ps1 /connectors/json/src/test/resources/empty.json: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /runner/src/it/resources/counters.csv: -------------------------------------------------------------------------------- 1 | 1,2,42,0 -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | /changelog/README.md merge=union 2 | -------------------------------------------------------------------------------- /runner/src/it/resources/duplicates.csv: -------------------------------------------------------------------------------- 1 | pk,v 2 | 1,42 -------------------------------------------------------------------------------- /runner/src/it/resources/bad_header_empty.csv: -------------------------------------------------------------------------------- 1 | field,, 2 | A,B 3 | -------------------------------------------------------------------------------- /runner/src/it/resources/function-pk.csv: -------------------------------------------------------------------------------- 1 | pk,v 2 | 1,1 3 | 2,2 4 | -------------------------------------------------------------------------------- /runner/src/it/resources/bad_header_duplicate.csv: -------------------------------------------------------------------------------- 1 | field,field 2 | A,B 3 | -------------------------------------------------------------------------------- /io/src/test/resources/compression/test-file: -------------------------------------------------------------------------------- 1 | this is 2 | a 3 | test file 4 | -------------------------------------------------------------------------------- /runner/src/it/resources/custom-type.csv: -------------------------------------------------------------------------------- 1 | 0,0x80730003666f6f00806900040000002000 -------------------------------------------------------------------------------- /runner/src/it/resources/invalid-mapping.csv: -------------------------------------------------------------------------------- 1 | invalid, 2 | ok1,1 3 | ok2,2 4 | -------------------------------------------------------------------------------- /connectors/csv/src/test/resources/bad_header_empty.csv: -------------------------------------------------------------------------------- 1 | field,,, 2 | A,B,C,D 3 | -------------------------------------------------------------------------------- /runner/src/it/resources/with-spaces.csv: -------------------------------------------------------------------------------- 1 | key,"my source" 2 | 1,"has 3 | new line" -------------------------------------------------------------------------------- /runner/src/it/resources/missing-extra.csv: -------------------------------------------------------------------------------- 1 | A,B,C 2 | 1,foo, 3 | 2,bar, 4 | 3,qix, 5 | -------------------------------------------------------------------------------- /workflow/api/src/main/resources/com/datastax/oss/dsbulk/version.txt: -------------------------------------------------------------------------------- 1 | ${project.version} -------------------------------------------------------------------------------- /config/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker: -------------------------------------------------------------------------------- 1 | mock-maker-inline -------------------------------------------------------------------------------- /connectors/csv/src/test/resources/bad_header_duplicate.csv: -------------------------------------------------------------------------------- 1 | field,field,field 2 | A,B,C 3 | -------------------------------------------------------------------------------- /runner/src/it/resources/empty-strings.json: -------------------------------------------------------------------------------- 1 | { 2 | "pk": "foo", 3 | "cc": "", 4 | "v": "" 5 | } -------------------------------------------------------------------------------- /runner/src/it/resources/numeric-fields.json: -------------------------------------------------------------------------------- 1 | { 2 | "0": 0, 3 | "1": 1, 4 | "2": 2 5 | } 6 | -------------------------------------------------------------------------------- /runner/src/test/resources/bad-json.conf: -------------------------------------------------------------------------------- 1 | dsbulk.connector.csv.url="C:\Users\cassandra\csv\badurl" -------------------------------------------------------------------------------- /workflow/commons/src/test/resources/application.conf: -------------------------------------------------------------------------------- 1 | dsbulk.log.directory = "./target/logs" 2 | -------------------------------------------------------------------------------- /runner/src/it/resources/empty-blobs.json: -------------------------------------------------------------------------------- 1 | { 2 | "pk": "0xcafebabe", 3 | "cc": "", 4 | "v": "" 5 | } -------------------------------------------------------------------------------- /runner/src/it/resources/temporal-numeric.csv: -------------------------------------------------------------------------------- 1 | key,vdate,vtime,vtimestamp 2 | 0 ,11520,123 ,123456 3 | -------------------------------------------------------------------------------- /runner/src/it/resources/with-spaces.json: -------------------------------------------------------------------------------- 1 | { 2 | "key": 1, 3 | "my source": "has\nnew line" 4 | } 5 | -------------------------------------------------------------------------------- /workflow/commons/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker: -------------------------------------------------------------------------------- 1 | mock-maker-inline -------------------------------------------------------------------------------- /runner/src/it/resources/comments.json: -------------------------------------------------------------------------------- 1 | { 2 | // this is a comment line 3 | "key": 1, 4 | "value": 1.0 5 | } 6 | -------------------------------------------------------------------------------- /connectors/csv/src/test/resources/multi-char-delimiter.csv: -------------------------------------------------------------------------------- 1 | field A || field B || field C 2 | foo |||bar| || "foo||bar" -------------------------------------------------------------------------------- /io/src/test/resources/compression/test.br: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/io/src/test/resources/compression/test.br -------------------------------------------------------------------------------- /io/src/test/resources/compression/test.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/io/src/test/resources/compression/test.gz -------------------------------------------------------------------------------- /io/src/test/resources/compression/test.xz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/io/src/test/resources/compression/test.xz -------------------------------------------------------------------------------- /io/src/test/resources/compression/test.z: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/io/src/test/resources/compression/test.z -------------------------------------------------------------------------------- /io/src/test/resources/compression/test.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/io/src/test/resources/compression/test.bz2 -------------------------------------------------------------------------------- /io/src/test/resources/compression/test.lz4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/io/src/test/resources/compression/test.lz4 -------------------------------------------------------------------------------- /io/src/test/resources/compression/test.lzma: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/io/src/test/resources/compression/test.lzma -------------------------------------------------------------------------------- /io/src/test/resources/compression/test.zstd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/io/src/test/resources/compression/test.zstd -------------------------------------------------------------------------------- /connectors/csv/src/test/resources/sample.csv.z: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/connectors/csv/src/test/resources/sample.csv.z -------------------------------------------------------------------------------- /io/src/test/resources/compression/test.snappy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/io/src/test/resources/compression/test.snappy -------------------------------------------------------------------------------- /tests/src/main/resources/ssl/client.keystore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/tests/src/main/resources/ssl/client.keystore -------------------------------------------------------------------------------- /tests/src/main/resources/ssl/client.truststore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/tests/src/main/resources/ssl/client.truststore -------------------------------------------------------------------------------- /tests/src/main/resources/ssl/server.keystore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/tests/src/main/resources/ssl/server.keystore -------------------------------------------------------------------------------- /tests/src/main/resources/ssl/server.truststore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/tests/src/main/resources/ssl/server.truststore -------------------------------------------------------------------------------- /connectors/csv/src/test/resources/sample.csv.br: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/connectors/csv/src/test/resources/sample.csv.br -------------------------------------------------------------------------------- /connectors/csv/src/test/resources/sample.csv.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/connectors/csv/src/test/resources/sample.csv.bz2 -------------------------------------------------------------------------------- /connectors/csv/src/test/resources/sample.csv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/connectors/csv/src/test/resources/sample.csv.gz -------------------------------------------------------------------------------- /connectors/csv/src/test/resources/sample.csv.lz4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/connectors/csv/src/test/resources/sample.csv.lz4 -------------------------------------------------------------------------------- /connectors/csv/src/test/resources/sample.csv.xz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/connectors/csv/src/test/resources/sample.csv.xz -------------------------------------------------------------------------------- /runner/src/it/resources/META-INF/services/com.datastax.oss.dsbulk.connectors.api.Connector: -------------------------------------------------------------------------------- 1 | com.datastax.oss.dsbulk.runner.tests.MockConnector 2 | -------------------------------------------------------------------------------- /runner/src/it/resources/ttl-timestamp.csv: -------------------------------------------------------------------------------- 1 | key, value, created_at ,time_to_live 2 | 1 ,bar ,"2017-11-29T14:32:15+02:00",1000 3 | -------------------------------------------------------------------------------- /connectors/csv/src/test/resources/sample.csv.lzma: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/connectors/csv/src/test/resources/sample.csv.lzma -------------------------------------------------------------------------------- /connectors/csv/src/test/resources/sample.csv.snappy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/connectors/csv/src/test/resources/sample.csv.snappy -------------------------------------------------------------------------------- /connectors/csv/src/test/resources/sample.csv.zstd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/connectors/csv/src/test/resources/sample.csv.zstd -------------------------------------------------------------------------------- /connectors/json/src/test/resources/multi_doc.json.z: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/connectors/json/src/test/resources/multi_doc.json.z -------------------------------------------------------------------------------- /runner/src/it/resources/ip-by-country-unique.csv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/runner/src/it/resources/ip-by-country-unique.csv.gz -------------------------------------------------------------------------------- /connectors/csv/src/main/resources/META-INF/services/com.datastax.oss.dsbulk.connectors.api.Connector: -------------------------------------------------------------------------------- 1 | com.datastax.oss.dsbulk.connectors.csv.CSVConnector 2 | -------------------------------------------------------------------------------- /connectors/json/src/test/resources/multi_doc.json.br: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/connectors/json/src/test/resources/multi_doc.json.br -------------------------------------------------------------------------------- /connectors/json/src/test/resources/multi_doc.json.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/connectors/json/src/test/resources/multi_doc.json.bz2 -------------------------------------------------------------------------------- /connectors/json/src/test/resources/multi_doc.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/connectors/json/src/test/resources/multi_doc.json.gz -------------------------------------------------------------------------------- /connectors/json/src/test/resources/multi_doc.json.lz4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/connectors/json/src/test/resources/multi_doc.json.lz4 -------------------------------------------------------------------------------- /connectors/json/src/test/resources/multi_doc.json.xz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/connectors/json/src/test/resources/multi_doc.json.xz -------------------------------------------------------------------------------- /config/src/test/resources/application.conf: -------------------------------------------------------------------------------- 1 | # Dummy application.conf used in ConfigUtilsTest 2 | dsbulk { 3 | definedInApplication = definedInApplication 4 | } -------------------------------------------------------------------------------- /connectors/json/src/main/resources/META-INF/services/com.datastax.oss.dsbulk.connectors.api.Connector: -------------------------------------------------------------------------------- 1 | com.datastax.oss.dsbulk.connectors.json.JsonConnector 2 | -------------------------------------------------------------------------------- /connectors/json/src/test/resources/multi_doc.json.lzma: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/connectors/json/src/test/resources/multi_doc.json.lzma -------------------------------------------------------------------------------- /connectors/json/src/test/resources/multi_doc.json.snappy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/connectors/json/src/test/resources/multi_doc.json.snappy -------------------------------------------------------------------------------- /connectors/json/src/test/resources/multi_doc.json.zstd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/connectors/json/src/test/resources/multi_doc.json.zstd -------------------------------------------------------------------------------- /tests/src/main/resources/ssl/server_localhost.keystore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datastax/dsbulk/HEAD/tests/src/main/resources/ssl/server_localhost.keystore -------------------------------------------------------------------------------- /workflow/load/src/main/resources/META-INF/services/com.datastax.oss.dsbulk.workflow.api.WorkflowProvider: -------------------------------------------------------------------------------- 1 | com.datastax.oss.dsbulk.workflow.load.LoadWorkflowProvider -------------------------------------------------------------------------------- /workflow/count/src/main/resources/META-INF/services/com.datastax.oss.dsbulk.workflow.api.WorkflowProvider: -------------------------------------------------------------------------------- 1 | com.datastax.oss.dsbulk.workflow.count.CountWorkflowProvider -------------------------------------------------------------------------------- /workflow/unload/src/main/resources/META-INF/services/com.datastax.oss.dsbulk.workflow.api.WorkflowProvider: -------------------------------------------------------------------------------- 1 | com.datastax.oss.dsbulk.workflow.unload.UnloadWorkflowProvider -------------------------------------------------------------------------------- /config/src/test/resources/application-custom.conf: -------------------------------------------------------------------------------- 1 | # Dummy application.conf used in ConfigUtilsTest 2 | dsbulk { 3 | definedInCustomApplication = definedInCustomApplication 4 | } -------------------------------------------------------------------------------- /runner/src/it/resources/temporal.csv: -------------------------------------------------------------------------------- 1 | key,vdate ,vtime ,vtimestamp 2 | 0 ,"vendredi, 9 mars 2018",171232584,2018-03-09T17:12:32.000+01:00[Europe/Paris] 3 | -------------------------------------------------------------------------------- /workflow/commons/src/main/resources/META-INF/services/com.datastax.oss.dsbulk.workflow.api.config.ConfigPostProcessor: -------------------------------------------------------------------------------- 1 | com.datastax.oss.dsbulk.workflow.commons.settings.PasswordPrompter -------------------------------------------------------------------------------- /batcher/reactor/src/main/resources/META-INF/services/com.datastax.oss.dsbulk.batcher.api.ReactiveStatementBatcherFactory: -------------------------------------------------------------------------------- 1 | com.datastax.oss.dsbulk.batcher.reactor.ReactorStatementBatcherFactory -------------------------------------------------------------------------------- /executor/reactor/src/main/resources/META-INF/services/com.datastax.oss.dsbulk.executor.api.BulkExecutorBuilderFactory: -------------------------------------------------------------------------------- 1 | com.datastax.oss.dsbulk.executor.reactor.ReactorBulkExecutorBuilderFactory -------------------------------------------------------------------------------- /runner/src/it/resources/temporal.json: -------------------------------------------------------------------------------- 1 | { 2 | "key": 0, 3 | "vdate": "vendredi, 9 mars 2018", 4 | "vtime": 171232584, 5 | "vtimestamp": "2018-03-09T17:12:32.000+01:00[Europe/Paris]" 6 | } -------------------------------------------------------------------------------- /runner/src/it/resources/ttl-timestamp-unset.csv: -------------------------------------------------------------------------------- 1 | key, value, created_at ,time_to_live 2 | 1 ,foo , , 3 | 2 ,bar ,"2017-11-29T14:32:15+02:00",1000 4 | -------------------------------------------------------------------------------- /config/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader TypeSafe Config Utilities 2 | 3 | This module contains utilities for reading and manipulating DSBulk's configuration, using the 4 | TypeSafe Config library. 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .settings 2 | .DS_Store 3 | 4 | /.idea 5 | *.iml 6 | .classpath 7 | .project 8 | 9 | .java-version 10 | .python-version 11 | 12 | target/ 13 | dependency-reduced-pom.xml 14 | 15 | logs/ 16 | -------------------------------------------------------------------------------- /codecs/text/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader Codecs - Text 2 | 3 | This module contains implementations of the ConvertingCodec API for Strings and Json. 4 | Json conversion is done using FasterXML Jackson library. 5 | -------------------------------------------------------------------------------- /url/src/main/resources/META-INF/services/com.datastax.oss.dsbulk.url.URLStreamHandlerProvider: -------------------------------------------------------------------------------- 1 | com.datastax.oss.dsbulk.url.StdinStdoutURLStreamHandlerProvider 2 | com.datastax.oss.dsbulk.url.S3URLStreamHandlerProvider -------------------------------------------------------------------------------- /runner/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader Runner 2 | 3 | This module contains the DSBulk's runner, and a parser for command lines. 4 | 5 | It automatically discovers all the available workflows using the Service Loader API. -------------------------------------------------------------------------------- /runner/src/it/resources/missing-extra.json: -------------------------------------------------------------------------------- 1 | { 2 | "A": 1, 3 | "B": "foo" 4 | } 5 | { 6 | "A": 2, 7 | "B": "bar", 8 | "C": null 9 | } 10 | { 11 | "A": 3, 12 | "B": "qix", 13 | "D": null 14 | } 15 | -------------------------------------------------------------------------------- /workflow/api/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader Workflow API 2 | 3 | Workflows form a pluggable abstraction that allows DSBulk to execute virtually any kind of 4 | operation. 5 | 6 | This module contains the Workflow API. 7 | -------------------------------------------------------------------------------- /connectors/csv/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader CSV Connector 2 | 3 | Connectors form a pluggable abstraction that allows DSBulk to read and write to a variety of 4 | backends. 5 | 6 | This module contains the CSV connector. 7 | -------------------------------------------------------------------------------- /connectors/json/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader Json Connector 2 | 3 | Connectors form a pluggable abstraction that allows DSBulk to read and write to a variety of 4 | backends. 5 | 6 | This module contains the Json connector. 7 | -------------------------------------------------------------------------------- /workflow/count/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader Count Workflow 2 | 3 | Workflows form a pluggable abstraction that allows DSBulk to execute virtually any kind of 4 | operation. 5 | 6 | This module contains the Count Workflow. 7 | -------------------------------------------------------------------------------- /workflow/load/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader Load Workflow 2 | 3 | Workflows form a pluggable abstraction that allows DSBulk to execute virtually any kind of 4 | operation. 5 | 6 | This module contains the Load Workflow. 7 | -------------------------------------------------------------------------------- /connectors/api/README.md: -------------------------------------------------------------------------------- 1 | 2 | # DataStax Bulk Loader Connector API 3 | 4 | Connectors form a pluggable abstraction that allows DSBulk to read and write to a variety of 5 | backends. 6 | 7 | This module contains the Connector API. 8 | -------------------------------------------------------------------------------- /workflow/unload/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader Unload Workflow 2 | 3 | Workflows form a pluggable abstraction that allows DSBulk to execute virtually any kind of 4 | operation. 5 | 6 | This module contains the Unload Workflow. 7 | -------------------------------------------------------------------------------- /config/src/test/resources/reference.conf: -------------------------------------------------------------------------------- 1 | # Dummy reference.conf file used in tests 2 | 3 | overriddenInApplication = 1 4 | fromReference = 2 5 | fromReferenceButNull = null 6 | 7 | dsbulk { 8 | definedInReference = definedInReference 9 | } -------------------------------------------------------------------------------- /runner/src/it/resources/numbers-missing.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "key": "scientific_notation", 4 | "vdecimal": "1.0E+7" 5 | }, 6 | { 7 | "key": "regular_notation", 8 | "vdouble": 10000000, 9 | "vdecimal": 10000000 10 | } 11 | ] -------------------------------------------------------------------------------- /batcher/api/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader Batcher API 2 | 3 | The Batcher API allows DSBulk to group statements together in batches using different grouping 4 | criteria and sizing characteristics. 5 | 6 | This module contains the Batcher API itself. 7 | -------------------------------------------------------------------------------- /codecs/text/src/main/resources/META-INF/services/com.datastax.oss.dsbulk.codecs.api.ConvertingCodecProvider: -------------------------------------------------------------------------------- 1 | com.datastax.oss.dsbulk.codecs.text.string.StringConvertingCodecProvider 2 | com.datastax.oss.dsbulk.codecs.text.json.JsonNodeConvertingCodecProvider 3 | -------------------------------------------------------------------------------- /config/src/test/resources/dsbulk-reference.conf: -------------------------------------------------------------------------------- 1 | # Dummy dsbulk-reference.conf used in ConfigUtilsTest 2 | # Note: the true dsbulk-reference.conf file is defined in dsbulk-engine module. 3 | dsbulk { 4 | definedInDSBukReference = definedInDSBukReference 5 | } -------------------------------------------------------------------------------- /io/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader IO Utilities 2 | 3 | This module contains I/O utilities for reading and writing to files, including compressed ones. 4 | 5 | The compression utilities were contributed by our beloved [Alex Ott](https://github.com/alexott). -------------------------------------------------------------------------------- /connectors/commons/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader Connectors Commons 2 | 3 | Connectors form a pluggable abstraction that allows DSBulk to read and write to a variety of 4 | backends. 5 | 6 | This module contains common base classes for text-based connectors. 7 | -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader Test Utilities 2 | 3 | This module contains test utilities for DSBulk, among which: 4 | 5 | 1. CCM test infrastructure; 6 | 2. Simulacron test infrastructure; 7 | 3. JUnit Extensions for logging and standard input/output assertions; 8 | 4. Custom assertions. -------------------------------------------------------------------------------- /batcher/reactor/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader Batcher Reactor Implementation 2 | 3 | The Batcher API allows DSBulk to group statements together in batches using different grouping 4 | criteria and sizing characteristics. 5 | 6 | This module contains an implementation of the Batcher API using Reactor. 7 | -------------------------------------------------------------------------------- /cql/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader CQL Parser 2 | 3 | This module contains a lightweight ANTLR 4 grammar and parser for the CQL language. 4 | 5 | **Important**: this module is not a general-purpose CQL parser; it cannot parse the entire CQL 6 | grammar but only a subset of it that DSBulk is capable of handling. -------------------------------------------------------------------------------- /format/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader Formatting Utilities 2 | 3 | This module contains utilities to format common DataStax Java driver objects: statements and rows. 4 | These utilities are used for logging purposes, during workflow execution to print statements and 5 | rows that failed to be sent or retrieved. -------------------------------------------------------------------------------- /sampler/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader Data Size Sampler 2 | 3 | This module contains a data size estimator and a data size sampler for statements (writes) and 4 | rows (reads). 5 | 6 | It is used by the [Batcher API](../batcher/api/README.md) to perform batching by data size, and by 7 | different workflows, to calibrate the concurrency level. 8 | -------------------------------------------------------------------------------- /workflow/commons/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader Workflow Commons 2 | 3 | Workflows form a pluggable abstraction that allows DSBulk to execute virtually any kind of 4 | operation. 5 | 6 | This module contains common base classes for workflows, and especially configuration utilities 7 | shared by DSBulk's built-in workflows (load, unload and count). 8 | -------------------------------------------------------------------------------- /runner/src/it/resources/missing.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "pk": null, 4 | "cc": "cc", 5 | "v": "pk is null" 6 | }, 7 | { 8 | "cc": "cc", 9 | "v": "pk is missing" 10 | }, 11 | { 12 | "pk": "pk", 13 | "cc": null, 14 | "v": "cc is null" 15 | }, 16 | { 17 | "pk": "pk", 18 | "v": "cc is missing" 19 | } 20 | ] 21 | -------------------------------------------------------------------------------- /runner/src/it/resources/complex.csv: -------------------------------------------------------------------------------- 1 | 0,,42,"[2,\"\",2.7,\"2018-05-25T11:25:00Z\"]","{\"2018-05-25T11:25:00Z\":\"\"}","[\"2018-05-25T11:25:00Z\",\"2018-05-25T11:26:00Z\"]","[\"foo\",\"\"]","{\"f_tuple\":[2,\"\",2.7,\"2018-05-25T11:25:00Z\"],\"f_list\":[\"2018-05-25T11:25:00Z\",\"2018-05-25T11:26:00Z\"]}" 2 | 1,N/A,,"[null,null,null,null]",{},[],[],"{\"f_tuple\":null,\"f_list\":[]}" 3 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader Documentation 2 | 3 | This module generates DSBulk's documentation. 4 | 5 | The generated documentation includes: 6 | 7 | 1. DSBulk's [settings reference](../manual/settings.md); 8 | 2. Sample configuration files: 9 | 1. [main application configuration file](../manual/application.template.conf) 10 | 2. [driver configuration file](../manual/driver.template.conf) 11 | -------------------------------------------------------------------------------- /runner/src/it/resources/missing-case.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "PK": null, 4 | "CC": "CC", 5 | "V": "PK is null case sensitive" 6 | }, 7 | { 8 | "CC": "CC", 9 | "V": "PK is missing case sensitive" 10 | }, 11 | { 12 | "PK": "PK", 13 | "CC": null, 14 | "V": "CC is null case sensitive" 15 | }, 16 | { 17 | "PK": "PK", 18 | "V": "CC is missing case sensitive" 19 | } 20 | ] -------------------------------------------------------------------------------- /batcher/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader Batcher 2 | 3 | The Batcher API allows DSBulk to group statements together in batches using different grouping 4 | criteria and sizing characteristics. 5 | 6 | This module has two submodules: 7 | 8 | 1. The [dsbulk-batcher-api](./api) submodule contains the Batcher API. 9 | 2. The [dsbulk-batcher-reactor](./reactor) submodule contains an implementation of the Batcher API 10 | using Reactor. 11 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | environment: 2 | appveyor_build_worker_cloud: gce 3 | test_profile: default 4 | matrix: 5 | - java_version: 1.8.0 6 | platform: x64 7 | init: 8 | - git config --global core.autocrlf true 9 | install: 10 | - ps: .\appveyor.ps1 11 | build_script: 12 | - mvn install -DskipTests=true -B -V 13 | test_script: 14 | - mvn verify -B 15 | on_finish: 16 | - ps: .\uploadtests.ps1 17 | cache: 18 | - C:\Users\appveyor\.m2 19 | -------------------------------------------------------------------------------- /codecs/jdk/src/main/resources/META-INF/services/com.datastax.oss.dsbulk.codecs.api.ConvertingCodecProvider: -------------------------------------------------------------------------------- 1 | com.datastax.oss.dsbulk.codecs.jdk.bool.BooleanConvertingCodecsProvider 2 | com.datastax.oss.dsbulk.codecs.jdk.collection.CollectionConvertingCodecsProvider 3 | com.datastax.oss.dsbulk.codecs.jdk.map.MapConvertingCodecsProvider 4 | com.datastax.oss.dsbulk.codecs.jdk.number.NumericConvertingCodecsProvider 5 | com.datastax.oss.dsbulk.codecs.jdk.temporal.TemporalConvertingCodecsProvider -------------------------------------------------------------------------------- /runner/src/it/resources/metrics/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 5s 3 | scrape_timeout: 4s 4 | evaluation_interval: 5s 5 | scrape_configs: 6 | - job_name: gateway 7 | honor_timestamps: true 8 | metrics_path: /metrics 9 | scheme: http 10 | static_configs: 11 | - targets: 12 | - gateway:9091 13 | - job_name: dsbulk 14 | honor_timestamps: true 15 | metrics_path: /metrics 16 | scheme: http 17 | static_configs: 18 | - targets: 19 | - host.testcontainers.internal:8080 20 | -------------------------------------------------------------------------------- /codecs/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader Codecs 2 | 3 | This module groups together submodules related to codec handling in DSBulk: 4 | 5 | 1. The [dsbulk-codecs-api](./api) submodule contains an API for codec handling in DSBulk. 6 | 2. The [dsbulk-codecs-text](./text) submodule contains implementations of that API for plain text 7 | and Json. 8 | 1. The [dsbulk-codecs-jdk](./jdk) submodule contains implementations of that API for converting to 9 | and from common JDK types: Boolean, Number, Temporal, UUID, Collections, and some driver types 10 | (TupleValue and UdtValue). 11 | -------------------------------------------------------------------------------- /codecs/api/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader Codec API 2 | 3 | This module contains an API for codec handling in DSBulk: 4 | 5 | 1. `ConvertingCodec`: a subtype of `TypeCodec` that handles conversions between an "external" Java 6 | type and an "internal" one. 7 | 3. `ConvertingCodecProvider`: a provider for `ConvertingCodec`s; implementors can 8 | provide codecs by registering their providers with the Service Loader API. 9 | 2. `ConvertingCodecFactory`: a factory for `ConvertingCodec`s; it discovers 10 | `ConvertingCodecProvider` implementations through the Service Loader API. 11 | -------------------------------------------------------------------------------- /uploadtests.ps1: -------------------------------------------------------------------------------- 1 | $testResults=Get-ChildItem TEST-TestSuite.xml -Recurse 2 | 3 | Write-Host "Uploading test results." 4 | 5 | $url = "https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)" 6 | $wc = New-Object 'System.Net.WebClient' 7 | 8 | foreach ($testResult in $testResults) { 9 | try { 10 | Write-Host -ForegroundColor Green "Uploading $testResult -> $url." 11 | $wc.UploadFile($url, $testResult) 12 | } catch [Net.WebException] { 13 | Write-Host -ForegroundColor Red "Failed Uploading $testResult -> $url. $_" 14 | } 15 | } 16 | 17 | Write-Host "Done uploading test results." -------------------------------------------------------------------------------- /connectors/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader Connectors 2 | 3 | Connectors form a pluggable abstraction that allows DSBulk to read and write to a variety of 4 | backends. 5 | 6 | This module groups together submodules related to connectors: 7 | 8 | 1. The [dsbulk-connectors-api](./api) submodule contains the Connector API. 9 | 2. The [dsbulk-connectors-commons](./commons) submodule contains common base classes for text-based 10 | connectors. 11 | 3. The [dsbulk-connectors-csv](./csv) submodule contains the CSV connector. 12 | 4. The [dsbulk-connectors-json](./json) submodule contains the Json connector. 13 | -------------------------------------------------------------------------------- /connectors/csv/src/test/resources/sample.csv: -------------------------------------------------------------------------------- 1 | 2 | 3 | # This example was extracted from Wikipedia (en.wikipedia.org/wiki/Comma-separated_values) 4 | # 5 | # 2 double quotes ("") are used as the escape sequence for quoted fields, as per the RFC4180 standard 6 | # 7 | 8 | Year,Make,Model,Description,Price 9 | 1997,Ford,E350," ac, abs, moon ",3000.00 10 | 1999,Chevy,"Venture ""Extended Edition""","",4900.00 11 | 12 | # Look, a multi line value. And blank rows around it! 13 | 14 | 1996,Jeep,Grand Cherokee,"MUST SELL! 15 | air, moon roof, loaded",4799.00 16 | 1999,Chevy,"Venture ""Extended Edition, Very Large""",,5000.00 17 | ,,"Venture ""Extended Edition""","",4900.00 18 | 19 | -------------------------------------------------------------------------------- /codecs/jdk/src/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader Codecs - Text 2 | 3 | This module contains implementations of the ConvertingCodec API to convert to and from common JDK types: 4 | 5 | 1. Boolean 6 | 2. Numbers 7 | 3. Temporals (both Java Time API and java.util.Date) 8 | 4. Collections (Lists, Maps, Sets) 9 | 5. UUID 10 | 11 | Some of the above types may also be converted into driver types such ad `UdtValue` and `TupleValue`. 12 | 13 | DSBulk itself does not use this module, since, as the time of writing (May 2020), it can only 14 | convert to/from text (Strings and Json). However other projects rely on the conversions provided 15 | in this module, e.g. the DataStax Kafka Connector Sink. -------------------------------------------------------------------------------- /workflow/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader Workflows 2 | 3 | Workflows form a pluggable abstraction that allows DSBulk to execute virtually any kind of 4 | operation. 5 | 6 | This module groups together submodules related to workflows: 7 | 8 | 1. The [dsbulk-workflow-api](./api) submodule contains the Workflow API. 9 | 2. The [dsbulk-workflow-commons](./commons) submodule contains common base classes for workflows, 10 | and especially configuration utilities shared by DSBulk's built-in workflows (load, unload and 11 | count). 12 | 3. The [dsbulk-workflow-load](./load) submodule contains the Load Workflow. 13 | 4. The [dsbulk-workflow-unload](./unload) submodule contains the Unload Workflow. 14 | 5. The [dsbulk-workflow-count](./count) submodule contains the Count Workflow. 15 | -------------------------------------------------------------------------------- /executor/api/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader Executor API 2 | 3 | DSBulk's Bulk Executor API is a pluggable abstraction that allows DSBulk to execute queries using 4 | reactive programming. 5 | 6 | **Important**: this module exists solely because initially the DataStax Java driver did not contain 7 | an API to execute queries reactively. Now that the driver exposes this feature, this module and its 8 | submodules should be considered as deprecated. DSBulk might remove this API and its implementations 9 | entirely in the near future, and replace them by the driver's equivalent API. 10 | 11 | However, the Executor API includes some features that are not present in the driver, such as the 12 | Execution Listener API. These features and APIs are likely to remain. 13 | 14 | This module contains the Executor API itself. 15 | -------------------------------------------------------------------------------- /codecs/api/src/main/java/com/datastax/oss/dsbulk/codecs/api/format/geo/GeoFormatTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.codecs.api.format.geo; 17 | 18 | public class GeoFormatTest {} 19 | -------------------------------------------------------------------------------- /runner/src/it/resources/ip-by-country-unique-part-1.csv: -------------------------------------------------------------------------------- 1 | "209.170.125.208","209.170.125.223","3517611472","3517611487","SE","Sweden" 2 | "80.242.192.0","80.242.207.255","1358086144","1358090239","CH","Switzerland" 3 | "200.31.64.0","200.31.95.255","3357491200","3357499391","CO","Colombia" 4 | "203.190.128.0","203.190.159.255","3418259456","3418267647","IN","India" 5 | "207.132.98.0","207.133.255.255","3481559552","3481665535","US","United States" 6 | "66.178.81.200","66.178.81.207","1118982600","1118982607","A2","Satellite Provider" 7 | "213.152.115.0","213.152.117.255","3583537920","3583538687","IL","Israel" 8 | "62.146.105.136","62.146.109.207","1049782664","1049783759","DE","Germany" 9 | "212.63.180.20","212.63.180.23","3560944660","3560944663","MZ","Mozambique" 10 | "195.79.12.0","195.79.12.79","3276737536","3276737615","IT","Italy" -------------------------------------------------------------------------------- /tests/src/main/java/com/datastax/oss/dsbulk/tests/logging/StreamType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.tests.logging; 17 | 18 | public enum StreamType { 19 | STDOUT, 20 | STDERR 21 | } 22 | -------------------------------------------------------------------------------- /runner/src/it/resources/part_1_csv/ip-by-country-unique-part-1.csv: -------------------------------------------------------------------------------- 1 | "209.170.125.208","209.170.125.223","3517611472","3517611487","SE","Sweden" 2 | "80.242.192.0","80.242.207.255","1358086144","1358090239","CH","Switzerland" 3 | "200.31.64.0","200.31.95.255","3357491200","3357499391","CO","Colombia" 4 | "203.190.128.0","203.190.159.255","3418259456","3418267647","IN","India" 5 | "207.132.98.0","207.133.255.255","3481559552","3481665535","US","United States" 6 | "66.178.81.200","66.178.81.207","1118982600","1118982607","A2","Satellite Provider" 7 | "213.152.115.0","213.152.117.255","3583537920","3583538687","IL","Israel" 8 | "62.146.105.136","62.146.109.207","1049782664","1049783759","DE","Germany" 9 | "212.63.180.20","212.63.180.23","3560944660","3560944663","MZ","Mozambique" 10 | "195.79.12.0","195.79.12.79","3276737536","3276737615","IT","Italy" -------------------------------------------------------------------------------- /runner/src/it/resources/application.conf: -------------------------------------------------------------------------------- 1 | dsbulk { 2 | 3 | # Mock Connector declaration. 4 | connector.mock {} 5 | 6 | log { 7 | 8 | directory = "./target/logs" 9 | 10 | } 11 | } 12 | 13 | datastax-java-driver { 14 | 15 | advanced { 16 | 17 | connection { 18 | init-query-timeout = 60 seconds 19 | set-keyspace-timeout = 60 seconds 20 | } 21 | 22 | control-connection.timeout = 60 seconds 23 | 24 | request { 25 | warn-if-set-keyspace = false 26 | } 27 | 28 | metrics { } 29 | 30 | // adjust quiet period to 0 seconds to speed up tests 31 | netty { 32 | io-group { 33 | shutdown {quiet-period = 0, timeout = 15, unit = SECONDS} 34 | } 35 | admin-group { 36 | shutdown {quiet-period = 0, timeout = 15, unit = SECONDS} 37 | } 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /executor/reactor/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader Executor Reactor Implementation 2 | 3 | DSBulk's Bulk Executor API is a pluggable abstraction that allows DSBulk to execute queries using 4 | reactive programming. 5 | 6 | **Important**: this module exists solely because initially the DataStax Java driver did not contain 7 | an API to execute queries reactively. Now that the driver exposes this feature, this module and its 8 | submodules should be considered as deprecated. DSBulk might remove this API and its implementations 9 | entirely in the near future, and replace them by the driver's equivalent API. 10 | 11 | However, the Executor API includes some features that are not present in the driver, such as the 12 | Execution Listener API. These features and APIs are likely to remain. 13 | 14 | This module contains an implementation of the Executor API using Reactor. 15 | -------------------------------------------------------------------------------- /mapping/src/main/java/com/datastax/oss/dsbulk/mapping/MappingToken.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.mapping; 17 | 18 | /** A token appearing in a mapping string, such as a field, or a CQL fragment. */ 19 | public interface MappingToken {} 20 | -------------------------------------------------------------------------------- /partitioner/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader Partitioning Utilities 2 | 3 | This module contains the `PartitionGenerator` API, that DSBulk uses to parallelize reads by 4 | splitting the table to read into a certain number of token range scans targeting the same replicas, 5 | which can then be fetched simultaneously from their respective replica sets. 6 | 7 | This API is inspired by the equivalent API from the Spark Connector. DSBulk's `PartitionGenerator` 8 | class, for instance, is adapted from Spark Connector's [`CassandraPartitionGenerator`], and other 9 | classes in this module have counterparts in the `com.datastax.spark.connector.rdd.partitioner` 10 | package in Spark Connector. 11 | 12 | [`CassandraPartitionGenerator`]: https://github.com/datastax/spark-cassandra-connector/blob/v2.4.3/spark-cassandra-connector/src/main/scala/com/datastax/spark/connector/rdd/partitioner/CassandraPartitionGenerator.scala -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | dist: trusty 3 | sudo: false 4 | # see https://sormuras.github.io/blog/2018-03-20-jdk-matrix.html 5 | matrix: 6 | include: 7 | # 8 8 | - env: JDK='OpenJDK 8' 9 | jdk: openjdk8 10 | # 11 11 | - env: JDK='OpenJDK 11' 12 | # switch to JDK 11 before running tests 13 | before_script: . $TRAVIS_BUILD_DIR/ci/install-jdk.sh -F 11 -L GPL 14 | # 12 15 | - env: JDK='OpenJDK 12' 16 | # switch to JDK 12 before running tests 17 | before_script: . $TRAVIS_BUILD_DIR/ci/install-jdk.sh -F 12 -L GPL 18 | 19 | before_install: 20 | - cp $TRAVIS_BUILD_DIR/ci/settings.xml $HOME/.m2/settings.xml 21 | # Require JDK8 for compiling 22 | - jdk_switcher use openjdk8 23 | 24 | install: mvn install -DskipTests -B -V 25 | 26 | script: mvn verify -B -V -Dmaven.main.skip=true -Dmaven.test.skip=true 27 | 28 | cache: 29 | directories: 30 | - $HOME/.m2 31 | 32 | -------------------------------------------------------------------------------- /connectors/json/src/test/resources/multi_doc.json: -------------------------------------------------------------------------------- 1 | /* 2 | This is a comment 3 | */ 4 | // doc1 5 | { 6 | "Year": 1997, 7 | "Make": "Ford", 8 | "Model": "E350", 9 | "Description": "ac, abs, moon", 10 | "Price": 3000.0 11 | } 12 | // doc2 13 | { 14 | "Year": 1999, 15 | "Make": "Chevy", 16 | "Model": "Venture \"Extended Edition\"", 17 | "Description": null, 18 | "Price": 4900.0 19 | } 20 | // doc3 21 | { 22 | "Year": 1996, 23 | "Make": "Jeep", 24 | "Model": "Grand Cherokee", 25 | "Description": "MUST SELL!\nair, moon roof, loaded", 26 | "Price": 4799.0 27 | } 28 | // doc4 29 | { 30 | "Year": 1999, 31 | "Make": "Chevy", 32 | "Model": "Venture \"Extended Edition, Very Large\"", 33 | "Description": null, 34 | "Price": 5000.0 35 | } 36 | // doc5 37 | { 38 | "Year": null, 39 | "Make": null, 40 | "Model": "Venture \"Extended Edition\"", 41 | "Description": null, 42 | "Price": 4900.0 43 | } 44 | -------------------------------------------------------------------------------- /runner/src/main/java/com/datastax/oss/dsbulk/runner/cli/VersionRequestException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.runner.cli; 17 | 18 | /** Simple exception indicating that the user wants to know the version of the tool. */ 19 | public class VersionRequestException extends Exception {} 20 | -------------------------------------------------------------------------------- /connectors/api/src/main/java/com/datastax/oss/dsbulk/connectors/api/ConnectorFeature.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.connectors.api; 17 | 18 | /** 19 | * A marker interface for connector features. 20 | * 21 | * @see Connector#supports(ConnectorFeature) 22 | */ 23 | public interface ConnectorFeature {} 24 | -------------------------------------------------------------------------------- /connectors/api/src/main/java/com/datastax/oss/dsbulk/connectors/api/IndexedField.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.connectors.api; 17 | 18 | /** A field in a {@link Record} identified by a zero-based integer index. */ 19 | public interface IndexedField extends Field { 20 | 21 | /** @return The field zero-based index. */ 22 | int getFieldIndex(); 23 | } 24 | -------------------------------------------------------------------------------- /connectors/json/src/test/resources/single_doc.json: -------------------------------------------------------------------------------- 1 | /* 2 | This is a comment 3 | */ 4 | [ 5 | // doc1 6 | { 7 | "Year": 1997, 8 | "Make": "Ford", 9 | "Model": "E350", 10 | "Description": "ac, abs, moon", 11 | "Price": 3000.0 12 | }, 13 | // doc2 14 | { 15 | "Year": 1999, 16 | "Make": "Chevy", 17 | "Model": "Venture \"Extended Edition\"", 18 | "Description": null, 19 | "Price": 4900.0 20 | }, 21 | // doc3 22 | { 23 | "Year": 1996, 24 | "Make": "Jeep", 25 | "Model": "Grand Cherokee", 26 | "Description": "MUST SELL!\nair, moon roof, loaded", 27 | "Price": 4799.0 28 | }, 29 | // doc4 30 | { 31 | "Year": 1999, 32 | "Make": "Chevy", 33 | "Model": "Venture \"Extended Edition, Very Large\"", 34 | "Description": null, 35 | "Price": 5000.0 36 | }, 37 | // doc5 38 | { 39 | "Year": null, 40 | "Make": null, 41 | "Model": "Venture \"Extended Edition\"", 42 | "Description": null, 43 | "Price": 4900.0 44 | } 45 | ] 46 | -------------------------------------------------------------------------------- /mapping/src/main/java/com/datastax/oss/dsbulk/mapping/MappingPreference.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.mapping; 17 | 18 | /** 19 | * How to determine if a mapping should be considered indexed or mapped in case of ambiguities, and 20 | * according to the connector's capabilities. 21 | */ 22 | public enum MappingPreference { 23 | MAPPED_ONLY, 24 | INDEXED_ONLY, 25 | MAPPED_OR_INDEXED 26 | } 27 | -------------------------------------------------------------------------------- /runner/src/it/resources/complex.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "pk": 0, 4 | "c_int": 42, 5 | "c_list": [ 6 | "2018-05-25T11:25:00Z", 7 | "2018-05-25T11:26:00Z" 8 | ], 9 | "c_map": { 10 | "2018-05-25T11:25:00Z": "" 11 | }, 12 | "c_set": [ 13 | "foo", 14 | "" 15 | ], 16 | "c_text": null, 17 | "c_tuple": [ 18 | 2, 19 | "", 20 | 2.7, 21 | "2018-05-25T11:25:00Z" 22 | ], 23 | "c_udt": { 24 | "f_tuple": [ 25 | 2, 26 | "", 27 | 2.7, 28 | "2018-05-25T11:25:00Z" 29 | ], 30 | "f_list": [ 31 | "2018-05-25T11:25:00Z", 32 | "2018-05-25T11:26:00Z" 33 | ] 34 | } 35 | }, 36 | { 37 | "pk": 1, 38 | "c_int": null, 39 | "c_list": [], 40 | "c_map": {}, 41 | "c_set": [], 42 | "c_text": "N/A", 43 | "c_tuple": [ 44 | null, 45 | null, 46 | null, 47 | null 48 | ], 49 | "c_udt": { 50 | "f_tuple": null, 51 | "f_list": [] 52 | } 53 | } 54 | ] -------------------------------------------------------------------------------- /connectors/api/src/main/java/com/datastax/oss/dsbulk/connectors/api/MappedField.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.connectors.api; 17 | 18 | import edu.umd.cs.findbugs.annotations.NonNull; 19 | 20 | /** A field in a {@link Record} identified by an alphanumeric name. */ 21 | public interface MappedField extends Field { 22 | 23 | /** @return The field name. */ 24 | @NonNull 25 | String getFieldName(); 26 | } 27 | -------------------------------------------------------------------------------- /executor/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader Executor 2 | 3 | DSBulk's Bulk Executor API is a pluggable abstraction that allows DSBulk to execute queries using 4 | reactive programming. 5 | 6 | **Important**: this module exists solely because initially the DataStax Java driver did not contain 7 | an API to execute queries reactively. Now that the driver exposes this feature, this module and its 8 | submodules should be considered as deprecated. DSBulk might remove this API and its implementations 9 | entirely in the near future, and replace them by the driver's equivalent API. 10 | 11 | However, the Executor API includes some features that are not present in the driver, such as the 12 | Execution Listener API. These features and APIs are likely to remain. 13 | 14 | This module groups together submodules related to reactive execution of queries: 15 | 16 | 1. The [dsbulk-executor-api](./api) submodule contains the Executor API. 17 | 2. The [dsbulk-executor-reactor](./reactor) submodule contains an implementation of the Executor API 18 | using Reactor. 19 | -------------------------------------------------------------------------------- /format/src/main/java/com/datastax/oss/dsbulk/format/statement/DefaultStatementPrinter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.format.statement; 17 | 18 | import com.datastax.oss.driver.api.core.cql.Statement; 19 | 20 | public class DefaultStatementPrinter extends StatementPrinterBase { 21 | 22 | @Override 23 | public Class getSupportedStatementClass() { 24 | return Statement.class; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /io/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | %-5level [%thread] %logger{40} - %msg%n 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /url/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | %-5level [%thread] %logger{40} - %msg%n 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /codecs/api/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | %-5level [%thread] %logger{40} - %msg%n 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /codecs/jdk/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | %-5level [%thread] %logger{40} - %msg%n 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /config/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | %-5level [%thread] %logger{40} - %msg%n 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /format/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | %-5level [%thread] %logger{40} - %msg%n 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /mapping/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | %-5level [%thread] %logger{40} - %msg%n 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /sampler/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | %-5level [%thread] %logger{40} - %msg%n 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /batcher/api/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | %-5level [%thread] %logger{40} - %msg%n 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /batcher/reactor/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | %-5level [%thread] %logger{40} - %msg%n 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /codecs/text/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | %-5level [%thread] %logger{40} - %msg%n 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /partitioner/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | %-5level [%thread] %logger{40} - %msg%n 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /executor/reactor/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | %-5level [%thread] %logger{40} - %msg%n 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /runner/src/main/java/com/datastax/oss/dsbulk/runner/cli/ParseException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.runner.cli; 17 | 18 | /** Simple exception indicating that the command line parsing failed. */ 19 | public class ParseException extends Exception { 20 | 21 | public ParseException(String message) { 22 | super(message); 23 | } 24 | 25 | public ParseException(String message, Throwable cause) { 26 | super(message, cause); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /workflow/commons/src/main/java/com/datastax/oss/dsbulk/workflow/commons/schema/NestedBatchException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.workflow.commons.schema; 17 | 18 | /** 19 | * Thrown when an attempt is made to include a BATCH child statement in a BATCH protocol message. 20 | */ 21 | public class NestedBatchException extends IllegalStateException { 22 | 23 | public NestedBatchException(String message) { 24 | super(message); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /workflow/commons/src/main/java/com/datastax/oss/dsbulk/workflow/commons/schema/ReadResultMapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.workflow.commons.schema; 17 | 18 | import com.datastax.oss.dsbulk.connectors.api.Record; 19 | import com.datastax.oss.dsbulk.executor.api.result.ReadResult; 20 | import edu.umd.cs.findbugs.annotations.NonNull; 21 | 22 | public interface ReadResultMapper { 23 | 24 | @NonNull 25 | Record map(@NonNull ReadResult result); 26 | } 27 | -------------------------------------------------------------------------------- /mapping/src/main/java/com/datastax/oss/dsbulk/mapping/IndexedMappingField.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.mapping; 17 | 18 | import com.datastax.oss.dsbulk.connectors.api.DefaultIndexedField; 19 | 20 | /** A field in a mapping declaration, identified by a zero-based integer index. */ 21 | public class IndexedMappingField extends DefaultIndexedField implements MappingField { 22 | 23 | public IndexedMappingField(int index) { 24 | super(index); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /runner/src/it/resources/ip-by-country-unique-part-2.csv: -------------------------------------------------------------------------------- 1 | "72.12.128.0","72.12.159.255","1208778752","1208786943","CA","Canada" 2 | "81.31.194.4","81.31.194.7","1361035780","1361035783","KW","Kuwait" 3 | "195.213.251.112","195.213.251.255","3285580656","3285580799","BE","Belgium" 4 | "195.212.32.0","195.212.34.255","3285458944","3285459711","DK","Denmark" 5 | "195.80.128.0","195.80.159.255","3276832768","3276840959","FR","France" 6 | "64.49.211.192","64.49.211.199","1077007296","1077007303","AU","Australia" 7 | "195.119.131.0","195.119.132.15","3279389440","3279389711","EU","Europe" 8 | "193.251.145.26","193.251.145.26","3254489370","3254489370","MR","Mauritania" 9 | "202.20.114.0","202.20.116.255","3390337536","3390338303","SG","Singapore" 10 | "65.75.129.220","65.75.129.227","1095467484","1095467491","TR","Turkey" 11 | "193.111.47.0","193.111.47.255","3245289216","3245289471","AT","Austria" 12 | "207.209.96.0","207.209.96.255","3486605312","3486605567","NL","Netherlands" 13 | "213.236.120.96","213.236.127.255","3589044320","3589046271","ES","Spain" 14 | "202.162.160.0","202.162.175.255","3399655424","3399659519","PH","Philippines" 15 | -------------------------------------------------------------------------------- /tests/src/main/java/com/datastax/oss/dsbulk/tests/logging/StreamCapture.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.tests.logging; 17 | 18 | import java.lang.annotation.ElementType; 19 | import java.lang.annotation.Retention; 20 | import java.lang.annotation.RetentionPolicy; 21 | import java.lang.annotation.Target; 22 | 23 | @Retention(RetentionPolicy.RUNTIME) 24 | @Target(ElementType.PARAMETER) 25 | public @interface StreamCapture { 26 | 27 | StreamType value() default StreamType.STDOUT; 28 | } 29 | -------------------------------------------------------------------------------- /mapping/src/main/java/com/datastax/oss/dsbulk/mapping/MappingField.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.mapping; 17 | 18 | import com.datastax.oss.dsbulk.connectors.api.Field; 19 | 20 | /** 21 | * A field in a mapping definition. Fields can be {@linkplain IndexedMappingField indexed} or 22 | * {@linkplain MappedMappingField named} (mapped). In a mapping definition, they appear on the left 23 | * side of a mapping entry. 24 | */ 25 | public interface MappingField extends MappingToken, Field {} 26 | -------------------------------------------------------------------------------- /runner/src/it/resources/part_2_csv/ip-by-country-unique-part-2.csv: -------------------------------------------------------------------------------- 1 | "72.12.128.0","72.12.159.255","1208778752","1208786943","CA","Canada" 2 | "81.31.194.4","81.31.194.7","1361035780","1361035783","KW","Kuwait" 3 | "195.213.251.112","195.213.251.255","3285580656","3285580799","BE","Belgium" 4 | "195.212.32.0","195.212.34.255","3285458944","3285459711","DK","Denmark" 5 | "195.80.128.0","195.80.159.255","3276832768","3276840959","FR","France" 6 | "64.49.211.192","64.49.211.199","1077007296","1077007303","AU","Australia" 7 | "195.119.131.0","195.119.132.15","3279389440","3279389711","EU","Europe" 8 | "193.251.145.26","193.251.145.26","3254489370","3254489370","MR","Mauritania" 9 | "202.20.114.0","202.20.116.255","3390337536","3390338303","SG","Singapore" 10 | "65.75.129.220","65.75.129.227","1095467484","1095467491","TR","Turkey" 11 | "193.111.47.0","193.111.47.255","3245289216","3245289471","AT","Austria" 12 | "207.209.96.0","207.209.96.255","3486605312","3486605567","NL","Netherlands" 13 | "213.236.120.96","213.236.127.255","3589044320","3589046271","ES","Spain" 14 | "202.162.160.0","202.162.175.255","3399655424","3399659519","PH","Philippines" 15 | -------------------------------------------------------------------------------- /partitioner/src/test/java/com/datastax/oss/dsbulk/partitioner/assertions/PartitionerAssertions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.partitioner.assertions; 17 | 18 | import com.datastax.oss.dsbulk.partitioner.BulkTokenRange; 19 | import com.datastax.oss.dsbulk.tests.assertions.TestAssertions; 20 | 21 | public class PartitionerAssertions extends TestAssertions { 22 | 23 | public static TokenRangeAssert assertThat(BulkTokenRange actual) { 24 | return new TokenRangeAssert(actual); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /runner/src/it/resources/graph/customer-orders.csv: -------------------------------------------------------------------------------- 1 | Customerid|Orderid 2 | 10000000-0000-0000-0000-000000000001|40000000-0000-0000-0001-000000000001 3 | 10000000-0000-0000-0000-000000000002|40000000-0000-0000-0002-000000000002 4 | 10000000-0000-0000-0000-000000000003|40000000-0000-0000-0003-000000000003 5 | 10000000-0000-0000-0000-000000000025|40000000-0000-0000-0025-000000000004 6 | 10000000-0000-0000-0000-000000000004|40000000-0000-0000-0004-000000000005 7 | 10000000-0000-0000-0000-000000000026|40000000-0000-0000-4021-000000000006 8 | 10000000-0000-0000-0000-000000000027|40000000-0000-0000-0991-000000000008 9 | 10000000-0000-0000-0000-000000000028|40000000-0000-0000-0101-000000000012 10 | 10000000-0000-0000-0000-000000000029|40000000-0000-0000-0018-000000000094 11 | 10000000-0000-0000-0000-000000000030|40000000-0000-0000-0003-000000000188 12 | 10000000-0000-0000-0000-000000000031|40000000-0000-0000-9184-000000000032 13 | 10000000-0000-0000-0000-000000000032|40000000-0000-0000-1225-000000000311 14 | 10000000-0000-0000-0000-000000000033|40000000-0000-0000-9110-000000040001 15 | 10000000-0000-0000-0000-000000000034|40000000-0000-0000-0148-000000000304 -------------------------------------------------------------------------------- /workflow/commons/src/main/java/com/datastax/oss/dsbulk/workflow/commons/schema/RecordMapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.workflow.commons.schema; 17 | 18 | import com.datastax.oss.driver.api.core.cql.BatchableStatement; 19 | import com.datastax.oss.dsbulk.connectors.api.Record; 20 | import edu.umd.cs.findbugs.annotations.NonNull; 21 | import reactor.core.publisher.Flux; 22 | 23 | public interface RecordMapper { 24 | 25 | @NonNull 26 | Flux> map(@NonNull Record record); 27 | } 28 | -------------------------------------------------------------------------------- /mapping/src/main/java/com/datastax/oss/dsbulk/mapping/MappedMappingField.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.mapping; 17 | 18 | import com.datastax.oss.dsbulk.connectors.api.DefaultMappedField; 19 | import edu.umd.cs.findbugs.annotations.NonNull; 20 | 21 | /** A field in a mapping definition identified by an alphanumeric name. */ 22 | public class MappedMappingField extends DefaultMappedField implements MappingField { 23 | 24 | public MappedMappingField(@NonNull String name) { 25 | super(name); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /executor/api/src/main/java/com/datastax/oss/dsbulk/executor/api/SyncBulkExecutor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.executor.api; 17 | 18 | import com.datastax.oss.dsbulk.executor.api.reader.SyncBulkReader; 19 | import com.datastax.oss.dsbulk.executor.api.writer.SyncBulkWriter; 20 | 21 | /** 22 | * An execution unit for {@link SyncBulkWriter bulk writes} and {@link SyncBulkReader bulk reads} 23 | * that operates in synchronous mode. 24 | */ 25 | public interface SyncBulkExecutor extends SyncBulkWriter, SyncBulkReader {} 26 | -------------------------------------------------------------------------------- /executor/api/src/main/java/com/datastax/oss/dsbulk/executor/api/AsyncBulkExecutor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.executor.api; 17 | 18 | import com.datastax.oss.dsbulk.executor.api.reader.AsyncBulkReader; 19 | import com.datastax.oss.dsbulk.executor.api.writer.AsyncBulkWriter; 20 | 21 | /** 22 | * An execution unit for {@link AsyncBulkWriter bulk writes} and {@link AsyncBulkReader bulk reads} 23 | * that operates in asynchronous mode. 24 | */ 25 | public interface AsyncBulkExecutor extends AsyncBulkWriter, AsyncBulkReader {} 26 | -------------------------------------------------------------------------------- /ci/settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | artifactory 7 | 8 | 9 | artifactory 10 | DataStax Artifactory 11 | 12 | true 13 | never 14 | warn 15 | 16 | 17 | true 18 | always 19 | fail 20 | 21 | https://repo.datastax.com/dse 22 | default 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | artifactory 33 | ${env.ARTIFACTORY_USERNAME} 34 | ${env.ARTIFACTORY_PASSWORD} 35 | 36 | 37 | 38 | 39 | 40 | artifactory 41 | 42 | 43 | -------------------------------------------------------------------------------- /format/src/main/java/com/datastax/oss/dsbulk/format/row/RowFormatterSymbols.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.format.row; 17 | 18 | class RowFormatterSymbols { 19 | 20 | static final String lineSeparator = System.lineSeparator(); 21 | static final String summaryStart = " ["; 22 | static final String summaryEnd = "]"; 23 | static final String valuesCount = "%s values"; 24 | static final String truncatedOutput = "..."; 25 | static final String nullValue = ""; 26 | static final String nameValueSeparator = ": "; 27 | } 28 | -------------------------------------------------------------------------------- /runner/src/main/java/com/datastax/oss/dsbulk/runner/ExitStatus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.runner; 17 | 18 | public enum ExitStatus { 19 | STATUS_OK(0), 20 | STATUS_COMPLETED_WITH_ERRORS(1), 21 | STATUS_ABORTED_TOO_MANY_ERRORS(2), 22 | STATUS_ABORTED_FATAL_ERROR(3), 23 | STATUS_INTERRUPTED(4), 24 | STATUS_CRASHED(5), 25 | ; 26 | 27 | private final int exitCode; 28 | 29 | ExitStatus(int exitCode) { 30 | this.exitCode = exitCode; 31 | } 32 | 33 | public int exitCode() { 34 | return exitCode; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /tests/src/main/java/com/datastax/oss/dsbulk/tests/utils/PlatformUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.tests.utils; 17 | 18 | public class PlatformUtils { 19 | 20 | /** 21 | * Checks if the operating system is a Windows one. 22 | * 23 | * @return true if the operating system is a Windows one, false 24 | * otherwise. 25 | */ 26 | public static boolean isWindows() { 27 | String osName = System.getProperty("os.name"); 28 | return osName != null && osName.startsWith("Windows"); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /executor/api/src/main/java/com/datastax/oss/dsbulk/executor/api/ReactiveBulkExecutor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.executor.api; 17 | 18 | import com.datastax.oss.dsbulk.executor.api.reader.ReactiveBulkReader; 19 | import com.datastax.oss.dsbulk.executor.api.writer.ReactiveBulkWriter; 20 | 21 | /** 22 | * An execution unit for {@link ReactiveBulkWriter bulk writes} and {@link ReactiveBulkReader bulk 23 | * reads} that operates in reactive mode. 24 | */ 25 | public interface ReactiveBulkExecutor extends ReactiveBulkWriter, ReactiveBulkReader {} 26 | -------------------------------------------------------------------------------- /workflow/commons/src/main/java/com/datastax/oss/dsbulk/workflow/commons/schema/ReadResultCounter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.workflow.commons.schema; 17 | 18 | import com.datastax.oss.dsbulk.executor.api.result.ReadResult; 19 | import java.io.IOException; 20 | 21 | public interface ReadResultCounter extends AutoCloseable { 22 | 23 | CountingUnit newCountingUnit(long initial); 24 | 25 | void reportTotals() throws IOException; 26 | 27 | interface CountingUnit extends AutoCloseable { 28 | 29 | void update(ReadResult result); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /mapping/src/main/java/com/datastax/oss/dsbulk/mapping/CQLFragment.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.mapping; 17 | 18 | /** 19 | * A fragment of CQL language included in a mapping definition or in a CQL query. Fragments can be 20 | * {@linkplain CQLWord CQL identifiers}, {@linkplain FunctionCall function calls}, or {@linkplain 21 | * CQLLiteral CQL literals}. In a mapping definition, they appear on the right side of a mapping 22 | * entry. 23 | */ 24 | public interface CQLFragment extends MappingToken { 25 | 26 | String render(CQLRenderMode mode); 27 | } 28 | -------------------------------------------------------------------------------- /connectors/api/src/main/java/com/datastax/oss/dsbulk/connectors/api/Field.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.connectors.api; 17 | 18 | import edu.umd.cs.findbugs.annotations.NonNull; 19 | 20 | /** 21 | * A field in a record. Fields can be {@linkplain IndexedField indexed} or {@linkplain MappedField 22 | * mapped}. 23 | */ 24 | public interface Field { 25 | 26 | /** 27 | * @return a generic description of the field, mainly for error reporting purposes; usually its 28 | * name or index. 29 | */ 30 | @NonNull 31 | String getFieldDescription(); 32 | } 33 | -------------------------------------------------------------------------------- /codecs/api/src/main/java/com/datastax/oss/dsbulk/codecs/api/format/geo/JsonGeoFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.codecs.api.format.geo; 17 | 18 | import com.datastax.dse.driver.api.core.data.geometry.Geometry; 19 | import edu.umd.cs.findbugs.annotations.Nullable; 20 | 21 | public class JsonGeoFormat implements GeoFormat { 22 | 23 | public static final JsonGeoFormat INSTANCE = new JsonGeoFormat(); 24 | 25 | private JsonGeoFormat() {} 26 | 27 | @Nullable 28 | @Override 29 | public String format(@Nullable Geometry geo) { 30 | return geo == null ? null : geo.asGeoJson(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /tests/src/main/java/com/datastax/oss/dsbulk/tests/logging/LogCapture.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.tests.logging; 17 | 18 | import java.lang.annotation.ElementType; 19 | import java.lang.annotation.Retention; 20 | import java.lang.annotation.RetentionPolicy; 21 | import java.lang.annotation.Target; 22 | import org.slf4j.event.Level; 23 | 24 | @Retention(RetentionPolicy.RUNTIME) 25 | @Target(ElementType.PARAMETER) 26 | public @interface LogCapture { 27 | 28 | Class value() default Root.class; 29 | 30 | String loggerName() default ""; 31 | 32 | Level level() default Level.INFO; 33 | 34 | class Root {} 35 | } 36 | -------------------------------------------------------------------------------- /codecs/api/src/main/java/com/datastax/oss/dsbulk/codecs/api/IdempotentConvertingCodec.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.codecs.api; 17 | 18 | import com.datastax.oss.driver.api.core.type.codec.TypeCodec; 19 | 20 | public class IdempotentConvertingCodec extends ConvertingCodec { 21 | 22 | public IdempotentConvertingCodec(TypeCodec internalCodec) { 23 | super(internalCodec, internalCodec.getJavaType()); 24 | } 25 | 26 | @Override 27 | public T externalToInternal(T value) { 28 | return value; 29 | } 30 | 31 | @Override 32 | public T internalToExternal(T value) { 33 | return value; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /codecs/api/src/main/java/com/datastax/oss/dsbulk/codecs/api/format/geo/WellKnownTextGeoFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.codecs.api.format.geo; 17 | 18 | import com.datastax.dse.driver.api.core.data.geometry.Geometry; 19 | import edu.umd.cs.findbugs.annotations.Nullable; 20 | 21 | public class WellKnownTextGeoFormat implements GeoFormat { 22 | 23 | public static final GeoFormat INSTANCE = new WellKnownTextGeoFormat(); 24 | 25 | private WellKnownTextGeoFormat() {} 26 | 27 | @Nullable 28 | @Override 29 | public String format(@Nullable Geometry geo) { 30 | return geo == null ? null : geo.asWellKnownText(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /config/src/main/java/com/datastax/oss/dsbulk/config/model/OrderedSettingsGroup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.config.model; 17 | 18 | import java.util.LinkedHashSet; 19 | import java.util.Set; 20 | 21 | /** A Group that contains settings ordered in the order they were added. */ 22 | class OrderedSettingsGroup implements SettingsGroup { 23 | 24 | private final Set settings = new LinkedHashSet<>(); 25 | 26 | @Override 27 | public boolean addSetting(String settingName) { 28 | return settings.add(settingName); 29 | } 30 | 31 | @Override 32 | public Set getSettings() { 33 | return settings; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /tests/src/main/java/com/datastax/oss/dsbulk/tests/ccm/annotations/CCMRequirements.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.tests.ccm.annotations; 17 | 18 | import static java.lang.annotation.ElementType.TYPE; 19 | import static java.lang.annotation.RetentionPolicy.RUNTIME; 20 | 21 | import com.datastax.oss.dsbulk.tests.ccm.CCMCluster.Type; 22 | import java.lang.annotation.Retention; 23 | import java.lang.annotation.Target; 24 | 25 | @Retention(RUNTIME) 26 | @Target(TYPE) 27 | public @interface CCMRequirements { 28 | 29 | Type[] compatibleTypes() default {Type.OSS, Type.DSE}; 30 | 31 | CCMVersionRequirement[] versionRequirements() default {}; 32 | } 33 | -------------------------------------------------------------------------------- /workflow/commons/src/main/java/com/datastax/oss/dsbulk/workflow/commons/log/RangeReadResource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.workflow.commons.log; 17 | 18 | import com.datastax.oss.dsbulk.executor.api.result.ReadResult; 19 | import edu.umd.cs.findbugs.annotations.NonNull; 20 | import java.net.URI; 21 | import org.reactivestreams.Publisher; 22 | 23 | /** 24 | * The equivalent of a connector Resource, but for range reads. Used to enable checkpointing in 25 | * unload and count workflows. 26 | */ 27 | public interface RangeReadResource { 28 | 29 | @NonNull 30 | URI getURI(); 31 | 32 | @NonNull 33 | Publisher read(); 34 | } 35 | -------------------------------------------------------------------------------- /config/src/main/java/com/datastax/oss/dsbulk/config/model/SettingsGroup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.config.model; 17 | 18 | import java.util.Set; 19 | 20 | /** 21 | * Encapsulates a group of settings that should be rendered together. When adding a setting to a 22 | * group, the add may be rejected because the given setting doesn't fit the criteria of settings 23 | * that belong to the group. This allows groups to act as listeners for settings and accept only 24 | * those that they are interested in. 25 | */ 26 | public interface SettingsGroup { 27 | 28 | boolean addSetting(String settingName); 29 | 30 | Set getSettings(); 31 | } 32 | -------------------------------------------------------------------------------- /codecs/api/src/main/java/com/datastax/oss/dsbulk/codecs/api/format/geo/GeoFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.codecs.api.format.geo; 17 | 18 | import com.datastax.dse.driver.api.core.data.geometry.Geometry; 19 | import edu.umd.cs.findbugs.annotations.Nullable; 20 | 21 | /** An utility for formatting geometry data. */ 22 | public interface GeoFormat { 23 | 24 | /** 25 | * Formats the given {@link Geometry} as a string. 26 | * 27 | * @param geo the value to format, may be {@code null}. 28 | * @return the formatted value or {@code null} if the value was {@code null}. 29 | */ 30 | @Nullable 31 | String format(@Nullable Geometry geo); 32 | } 33 | -------------------------------------------------------------------------------- /codecs/text/src/main/java/com/datastax/oss/dsbulk/codecs/text/string/StringToListCodec.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.codecs.text.string; 17 | 18 | import com.datastax.oss.dsbulk.codecs.api.ConvertingCodec; 19 | import com.fasterxml.jackson.databind.JsonNode; 20 | import com.fasterxml.jackson.databind.ObjectMapper; 21 | import java.util.List; 22 | 23 | public class StringToListCodec extends StringToCollectionCodec> { 24 | 25 | public StringToListCodec( 26 | ConvertingCodec> jsonCodec, 27 | ObjectMapper objectMapper, 28 | List nullStrings) { 29 | super(jsonCodec, objectMapper, nullStrings); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /workflow/api/src/main/java/com/datastax/oss/dsbulk/workflow/api/error/UnlimitedErrorThreshold.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.workflow.api.error; 17 | 18 | import edu.umd.cs.findbugs.annotations.NonNull; 19 | 20 | public class UnlimitedErrorThreshold implements ErrorThreshold { 21 | 22 | public static final UnlimitedErrorThreshold INSTANCE = new UnlimitedErrorThreshold(); 23 | 24 | private UnlimitedErrorThreshold() {} 25 | 26 | @Override 27 | public boolean checkThresholdExceeded(long errorCount, @NonNull Number totalItems) { 28 | return false; 29 | } 30 | 31 | @Override 32 | public String thresholdAsString() { 33 | return "unlimited"; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /codecs/text/src/main/java/com/datastax/oss/dsbulk/codecs/text/string/StringToSetCodec.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.codecs.text.string; 17 | 18 | import com.datastax.oss.dsbulk.codecs.api.ConvertingCodec; 19 | import com.fasterxml.jackson.databind.JsonNode; 20 | import com.fasterxml.jackson.databind.ObjectMapper; 21 | import java.util.List; 22 | import java.util.Set; 23 | 24 | public class StringToSetCodec extends StringToCollectionCodec> { 25 | 26 | public StringToSetCodec( 27 | ConvertingCodec> jsonCodec, 28 | ObjectMapper objectMapper, 29 | List nullStrings) { 30 | super(jsonCodec, objectMapper, nullStrings); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /executor/reactor/src/main/java/com/datastax/oss/dsbulk/executor/reactor/ReactorBulkExecutor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.executor.reactor; 17 | 18 | import com.datastax.oss.dsbulk.executor.api.BulkExecutor; 19 | import com.datastax.oss.dsbulk.executor.reactor.reader.ReactorBulkReader; 20 | import com.datastax.oss.dsbulk.executor.reactor.writer.ReactorBulkWriter; 21 | 22 | /** 23 | * An execution unit for {@link ReactorBulkWriter bulk writes} and {@link ReactorBulkReader bulk 24 | * reads} that operates in reactive mode using Reactor. 25 | */ 26 | public interface ReactorBulkExecutor extends ReactorBulkWriter, ReactorBulkReader, BulkExecutor {} 27 | -------------------------------------------------------------------------------- /runner/src/main/java/com/datastax/oss/dsbulk/runner/cli/GlobalHelpRequestException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.runner.cli; 17 | 18 | import edu.umd.cs.findbugs.annotations.Nullable; 19 | 20 | /** Simple exception indicating that the user wants the main help output. */ 21 | public class GlobalHelpRequestException extends Exception { 22 | 23 | private final String connectorName; 24 | 25 | GlobalHelpRequestException() { 26 | this(null); 27 | } 28 | 29 | GlobalHelpRequestException(@Nullable String connectorName) { 30 | this.connectorName = connectorName; 31 | } 32 | 33 | @Nullable 34 | public String getConnectorName() { 35 | return connectorName; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /url/src/test/java/com/datastax/oss/dsbulk/url/UncloseableOutputStreamTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.url; 17 | 18 | import static org.mockito.Mockito.never; 19 | import static org.mockito.Mockito.spy; 20 | import static org.mockito.Mockito.verify; 21 | 22 | import java.io.ByteArrayOutputStream; 23 | import org.junit.jupiter.api.Test; 24 | 25 | class UncloseableOutputStreamTest { 26 | 27 | @Test 28 | void should_not_close_output_stream() throws Exception { 29 | ByteArrayOutputStream delegate = spy(new ByteArrayOutputStream()); 30 | UncloseableOutputStream is = new UncloseableOutputStream(delegate); 31 | is.close(); 32 | verify(delegate, never()).close(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /tests/src/main/java/com/datastax/oss/dsbulk/tests/utils/MemoryUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.tests.utils; 17 | 18 | import com.sun.management.OperatingSystemMXBean; 19 | import java.lang.management.ManagementFactory; 20 | 21 | public class MemoryUtils { 22 | 23 | /** 24 | * Returns the system's free memory in megabytes. 25 | * 26 | *

This includes the free physical memory + the free swap memory. 27 | */ 28 | public static long getFreeMemoryMB() { 29 | OperatingSystemMXBean bean = 30 | (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); 31 | return (bean.getFreePhysicalMemorySize() + bean.getFreeSwapSpaceSize()) / 1024 / 1024; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /executor/api/src/main/java/com/datastax/oss/dsbulk/executor/api/BulkExecutor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.executor.api; 17 | 18 | import com.datastax.oss.dsbulk.executor.api.reader.BulkReader; 19 | import com.datastax.oss.dsbulk.executor.api.writer.BulkWriter; 20 | 21 | /** 22 | * An execution unit for {@link BulkWriter bulk writes} and {@link BulkReader bulk reads} that 23 | * operates in 3 distinct modes: {@link SyncBulkExecutor synchronous} (blocking), {@link 24 | * AsyncBulkExecutor asynchronous} (non-blocking), and {@link ReactiveBulkExecutor reactive}. 25 | */ 26 | public interface BulkExecutor 27 | extends BulkWriter, BulkReader, SyncBulkExecutor, AsyncBulkExecutor, ReactiveBulkExecutor {} 28 | -------------------------------------------------------------------------------- /runner/src/it/resources/number.csv: -------------------------------------------------------------------------------- 1 | key;vdouble;vdecimal 2 | 3 | # Scientific notation 4 | scientific_notation;1.0e+7;1.0e+7 5 | 6 | # 1e+7 in regular notation 7 | regular_notation;10000000;10000000 8 | 9 | # Hexadecimal floating point notation 10 | hex_notation;0x1.fffffffffffffP+1023;0x1.fffffffffffffP+1023 11 | 12 | # Cannot be represented exactly with IEEE 754 13 | irrational;0.1;0.1 14 | 15 | # Double.NaN 16 | Double.NaN;NaN; 17 | 18 | # Double.POSITIVE_INFINITY 19 | Double.POSITIVE_INFINITY;+Infinity; 20 | 21 | # Double.NEGATIVE_INFINITY 22 | Double.NEGATIVE_INFINITY;-Infinity; 23 | 24 | # Double.MAX_VALUE 25 | Double.MAX_VALUE;1.7976931348623157E308;1.7976931348623157E308 26 | 27 | # Double.MIN_VALUE 28 | Double.MIN_VALUE;4.9E-324;4.9E-324 29 | 30 | # Double.MIN_NORMAL 31 | Double.MIN_NORMAL;2.2250738585072014E-308;2.2250738585072014E-308 32 | 33 | # Float.MAX_VALUE, in regular notation 34 | Float.MAX_VALUE;340,282,350,000,000,000,000,000,000,000,000,000,000;340,282,350,000,000,000,000,000,000,000,000,000,000 35 | 36 | # Float.MIN_VALUE, in regular notation 37 | Float.MIN_VALUE;0.0000000000000000000000000000000000000000000014;0.0000000000000000000000000000000000000000000014 38 | 39 | # Too many precision digits (> 16), will be rejected or truncated 40 | too_many_digits;0.12345678901234567890123456789;0.12345678901234567890123456789 41 | 42 | -------------------------------------------------------------------------------- /url/README.md: -------------------------------------------------------------------------------- 1 | # DataStax Bulk Loader URL Utilities 2 | 3 | This module contains URL utilities for DSBulk, among which: 4 | 5 | 1. DSBulk's `BulkLoaderURLStreamHandlerFactory`, which is DSBulk's default factory for URL handlers; 6 | 2. A URL stream handler for reading / writing to standard input / output. 7 | 3. A URL stream handler for reading from AWS S3 URLs. 8 | 1. Every S3 URL must contain the proper query parameters from which an `S3Client` can be built. These parameters are: 9 | 1. `region` (required): The AWS region, such as `us-west-1`. 10 | 2. `profile` (optional, preferred): The profile to use to provide credentials. See [the AWS SDK credentials documentation](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/credentials.html) for more information. 11 | 3. `accessKeyId` and `secretKeyId` (optional, discouraged): In case you don't have a profile set up, you can use this less-secure method. Both parameters are required if you choose this. 12 | 2. If only the `region` is provided, DSBulk will fall back to the default AWS credentials provider, which handles role-based credentials. 13 | 3. To prevent unnecessary client re-creation when using many URLs from a `urlfile`, `S3Client`s are cached by the query parameters. The size of the cache is controlled by the `dsbulk.s3.clientCacheSize` option (default: 20). 14 | -------------------------------------------------------------------------------- /url/src/test/java/com/datastax/oss/dsbulk/url/UncloseableInputStreamTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.url; 17 | 18 | import static org.mockito.Mockito.never; 19 | import static org.mockito.Mockito.spy; 20 | import static org.mockito.Mockito.verify; 21 | 22 | import java.io.ByteArrayInputStream; 23 | import org.junit.jupiter.api.Test; 24 | 25 | class UncloseableInputStreamTest { 26 | 27 | @Test 28 | void should_not_close_input_stream() throws Exception { 29 | ByteArrayInputStream delegate = spy(new ByteArrayInputStream(new byte[] {1, 2, 3})); 30 | UncloseableInputStream is = new UncloseableInputStream(delegate); 31 | is.close(); 32 | verify(delegate, never()).close(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /connectors/api/src/main/java/com/datastax/oss/dsbulk/connectors/api/Resource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.connectors.api; 17 | 18 | import edu.umd.cs.findbugs.annotations.NonNull; 19 | import java.net.URI; 20 | import org.reactivestreams.Publisher; 21 | 22 | /** 23 | * A resource that can be read from or written to. It could be a file, a database table, etc. 24 | * 25 | *

Resources are sources of records, and are identified by a unique URI. 26 | */ 27 | public interface Resource { 28 | 29 | /** @return The URI of this resource. */ 30 | @NonNull 31 | URI getURI(); 32 | 33 | /** @return A publisher that will emit records from this resource. */ 34 | @NonNull 35 | Publisher read(); 36 | } 37 | -------------------------------------------------------------------------------- /codecs/api/src/main/java/com/datastax/oss/dsbulk/codecs/api/ConversionContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.codecs.api; 17 | 18 | import edu.umd.cs.findbugs.annotations.NonNull; 19 | import java.util.concurrent.ConcurrentHashMap; 20 | import java.util.concurrent.ConcurrentMap; 21 | 22 | public class ConversionContext { 23 | 24 | private final ConcurrentMap attributes = new ConcurrentHashMap<>(); 25 | 26 | public void addAttribute(@NonNull String key, Object value) { 27 | attributes.put(key, value); 28 | } 29 | 30 | @SuppressWarnings({"unchecked", "TypeParameterUnusedInFormals"}) 31 | public T getAttribute(@NonNull String key) { 32 | return (T) attributes.get(key); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /distribution/src/assembly/sources.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | sources 21 | 22 | jar 23 | 24 | false 25 | 26 | 27 | ${project.build.directory}/sources 28 | ./ 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /runner/src/main/java/com/datastax/oss/dsbulk/runner/cli/ParsedCommandLine.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.runner.cli; 17 | 18 | import com.datastax.oss.dsbulk.workflow.api.WorkflowProvider; 19 | import com.typesafe.config.Config; 20 | 21 | public class ParsedCommandLine { 22 | 23 | private final WorkflowProvider workflowProvider; 24 | private final Config config; 25 | 26 | ParsedCommandLine(WorkflowProvider workflowProvider, Config config) { 27 | this.workflowProvider = workflowProvider; 28 | this.config = config; 29 | } 30 | 31 | public WorkflowProvider getWorkflowProvider() { 32 | return workflowProvider; 33 | } 34 | 35 | public Config getConfig() { 36 | return config; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /workflow/commons/src/main/java/com/datastax/oss/dsbulk/workflow/commons/utils/StringUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.workflow.commons.utils; 17 | 18 | public class StringUtils { 19 | 20 | /** 21 | * Left pad a string with spaces to a size of {@code size}. 22 | * 23 | * @param s String to pad. 24 | * @param size the size. 25 | * @return the padded string. 26 | */ 27 | public static String leftPad(String s, int size) { 28 | int repeat = size - s.length(); 29 | if (repeat <= 0) { 30 | return s; 31 | } 32 | char[] buf = new char[repeat]; 33 | for (int i = 0; i < repeat; i++) { 34 | buf[i] = ' '; 35 | } 36 | return new String(buf).concat(s); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /io/src/test/java/com/datastax/oss/dsbulk/io/IOUtilsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.io; 17 | 18 | import static org.assertj.core.api.Assertions.assertThat; 19 | 20 | import com.datastax.oss.dsbulk.url.BulkLoaderURLStreamHandlerFactory; 21 | import java.net.MalformedURLException; 22 | import java.net.URL; 23 | import org.junit.jupiter.api.Test; 24 | 25 | class IOUtilsTest { 26 | 27 | static { 28 | BulkLoaderURLStreamHandlerFactory.install(); 29 | } 30 | 31 | @Test 32 | void should_detect_standard_stream_url() throws MalformedURLException { 33 | assertThat(IOUtils.isStandardStream(new URL("http://acme.com"))).isFalse(); 34 | assertThat(IOUtils.isStandardStream(new URL("std:/"))).isTrue(); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /appveyor.ps1: -------------------------------------------------------------------------------- 1 | Add-Type -AssemblyName System.IO.Compression.FileSystem 2 | 3 | $dep_dir="C:\Users\appveyor\deps" 4 | If (!(Test-Path $dep_dir)) { 5 | Write-Host "Creating $($dep_dir)" 6 | New-Item -Path $dep_dir -ItemType Directory -Force 7 | } 8 | 9 | $openssl_platform = "Win32" 10 | $vc_platform = "x86" 11 | If ($env:PLATFORM -eq "X64") { 12 | $vc_platform = "x64" 13 | } 14 | 15 | $env:JAVA_HOME="C:\Program Files\Java\jdk$($env:java_version)" 16 | # The configured java version to test with. 17 | 18 | $maven_base = "$($dep_dir)\maven" 19 | $maven_path = "$($maven_base)\apache-maven-3.2.5" 20 | If (!(Test-Path $maven_path)) { 21 | Write-Host "Installing Maven" 22 | $maven_url = "https://www.dropbox.com/s/fh9kffmexprsmha/apache-maven-3.2.5-bin.zip?raw=1" 23 | $maven_zip = "C:\Users\appveyor\apache-maven-3.2.5-bin.zip" 24 | (new-object System.Net.WebClient).DownloadFile($maven_url, $maven_zip) 25 | [System.IO.Compression.ZipFile]::ExtractToDirectory($maven_zip, $maven_base) 26 | } 27 | $env:M2_HOME="$($maven_path)" 28 | $env:PATH="$($maven_path)\bin;$($env:PATH)" 29 | 30 | # Environment variables for Publisher Verification tests (Reactive Streams TCK) 31 | $env:DEFAULT_TIMEOUT_MILLIS=1000 32 | $env:DEFAULT_NO_SIGNALS_TIMEOUT_MILLIS=1000 33 | $env:PUBLISHER_REFERENCE_GC_TIMEOUT_MILLIS=1000 34 | 35 | Copy-Item ./ci/settings.xml -Destination "$($maven_path)\conf\settings.xml" -------------------------------------------------------------------------------- /executor/reactor/src/main/java/com/datastax/oss/dsbulk/executor/reactor/DefaultReactorBulkExecutorBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.executor.reactor; 17 | 18 | import com.datastax.oss.driver.api.core.CqlSession; 19 | import com.datastax.oss.dsbulk.executor.api.AbstractBulkExecutorBuilder; 20 | 21 | /** A builder for {@link DefaultReactorBulkExecutor} instances. */ 22 | public class DefaultReactorBulkExecutorBuilder 23 | extends AbstractBulkExecutorBuilder { 24 | 25 | DefaultReactorBulkExecutorBuilder(CqlSession session) { 26 | super(session); 27 | } 28 | 29 | @Override 30 | public DefaultReactorBulkExecutor build() { 31 | return new DefaultReactorBulkExecutor(this); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /tests/src/main/java/com/datastax/oss/dsbulk/tests/logging/LogInterceptor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.tests.logging; 17 | 18 | import ch.qos.logback.classic.spi.ILoggingEvent; 19 | import java.util.List; 20 | import java.util.stream.Collectors; 21 | 22 | public interface LogInterceptor { 23 | 24 | List getLoggedEvents(); 25 | 26 | default List getLoggedMessages() { 27 | return getLoggedEvents().stream() 28 | .map(ILoggingEvent::getFormattedMessage) 29 | .collect(Collectors.toList()); 30 | } 31 | 32 | default String getAllMessagesAsString() { 33 | return getLoggedMessages().stream().collect(Collectors.joining(System.lineSeparator())); 34 | } 35 | 36 | void clear(); 37 | } 38 | -------------------------------------------------------------------------------- /workflow/commons/src/main/java/com/datastax/oss/dsbulk/workflow/commons/statement/MappedStatement.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.workflow.commons.statement; 17 | 18 | import com.datastax.oss.dsbulk.connectors.api.Record; 19 | import edu.umd.cs.findbugs.annotations.NonNull; 20 | 21 | /** 22 | * A statement that has been produced by mapping fields of a {@link Record} to variables in the 23 | * query. 24 | * 25 | * @see com.datastax.oss.dsbulk.workflow.commons.schema.DefaultRecordMapper 26 | */ 27 | public interface MappedStatement { 28 | 29 | /** 30 | * Returns the record that this statement was mapped from. 31 | * 32 | * @return the record of this statement. 33 | */ 34 | @NonNull 35 | Record getRecord(); 36 | } 37 | -------------------------------------------------------------------------------- /tests/src/main/java/com/datastax/oss/dsbulk/tests/driver/annotations/SessionFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.tests.driver.annotations; 17 | 18 | import static java.lang.annotation.ElementType.METHOD; 19 | import static java.lang.annotation.RetentionPolicy.RUNTIME; 20 | 21 | import java.lang.annotation.Retention; 22 | import java.lang.annotation.Target; 23 | 24 | /** 25 | * Annotation that marks a method as being a factory for {@link 26 | * com.datastax.oss.driver.api.core.CqlSession} instances. 27 | * 28 | *

Such methods must be static and their return type must be {@link 29 | * com.datastax.oss.driver.api.core.CqlSessionBuilder}. 30 | */ 31 | @Retention(RUNTIME) 32 | @Target(METHOD) 33 | public @interface SessionFactory {} 34 | -------------------------------------------------------------------------------- /tests/src/main/java/com/datastax/oss/dsbulk/tests/ccm/annotations/CCMFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.tests.ccm.annotations; 17 | 18 | import static java.lang.annotation.ElementType.METHOD; 19 | import static java.lang.annotation.RetentionPolicy.RUNTIME; 20 | 21 | import com.datastax.oss.dsbulk.tests.ccm.CCMCluster; 22 | import com.datastax.oss.dsbulk.tests.ccm.DefaultCCMCluster.Builder; 23 | import java.lang.annotation.Retention; 24 | import java.lang.annotation.Target; 25 | 26 | /** 27 | * Annotation that marks a method as being a factory for {@link CCMCluster} instances. 28 | * 29 | *

Such methods must be static and their return type must be {@link Builder}. 30 | */ 31 | @Retention(RUNTIME) 32 | @Target(METHOD) 33 | public @interface CCMFactory {} 34 | -------------------------------------------------------------------------------- /connectors/api/src/main/java/com/datastax/oss/dsbulk/connectors/api/DefaultResource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.connectors.api; 17 | 18 | import edu.umd.cs.findbugs.annotations.NonNull; 19 | import java.net.URI; 20 | import org.reactivestreams.Publisher; 21 | 22 | public class DefaultResource implements Resource { 23 | 24 | private final URI uri; 25 | private final Publisher records; 26 | 27 | public DefaultResource(@NonNull URI uri, @NonNull Publisher records) { 28 | this.uri = uri; 29 | this.records = records; 30 | } 31 | 32 | @NonNull 33 | @Override 34 | public URI getURI() { 35 | return uri; 36 | } 37 | 38 | @NonNull 39 | @Override 40 | public Publisher read() { 41 | return records; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /partitioner/src/main/java/com/datastax/oss/dsbulk/partitioner/BulkTokenRange.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.partitioner; 17 | 18 | import com.datastax.oss.driver.api.core.metadata.EndPoint; 19 | import com.datastax.oss.driver.api.core.metadata.token.TokenRange; 20 | import edu.umd.cs.findbugs.annotations.NonNull; 21 | import java.math.BigInteger; 22 | import java.util.Set; 23 | 24 | public interface BulkTokenRange extends TokenRange { 25 | 26 | /** @return The replicas that own this range. */ 27 | @NonNull 28 | Set replicas(); 29 | 30 | /** @return The range size (i.e. the number of tokens it contains). */ 31 | @NonNull 32 | BigInteger size(); 33 | 34 | /** @return The ring fraction covered by this range. */ 35 | double fraction(); 36 | } 37 | -------------------------------------------------------------------------------- /sampler/src/main/java/com/datastax/oss/dsbulk/sampler/Sizeable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.sampler; 17 | 18 | /** 19 | * A data container that can report the size of its contents. 20 | * 21 | *

This interface is used by {@link DataSizes} to determine the size of the data. It is meant to 22 | * be implemented by {@link com.datastax.oss.driver.api.core.cql.Row Row} and {@link 23 | * com.datastax.oss.driver.api.core.cql.Statement Statement} implementations wishing to optimize or 24 | * cache the data size computation. 25 | * 26 | * @see SizeableRow 27 | * @see SizeableBoundStatement 28 | * @see SizeableBatchStatement 29 | */ 30 | public interface Sizeable { 31 | 32 | /** @return the size of the container data in bytes. */ 33 | long getDataSize(); 34 | } 35 | -------------------------------------------------------------------------------- /tests/src/main/java/com/datastax/oss/dsbulk/tests/ccm/annotations/CCMWorkload.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.tests.ccm.annotations; 17 | 18 | import static java.lang.annotation.ElementType.ANNOTATION_TYPE; 19 | import static java.lang.annotation.RetentionPolicy.RUNTIME; 20 | 21 | import com.datastax.oss.dsbulk.tests.ccm.CCMCluster.Workload; 22 | import java.lang.annotation.Retention; 23 | import java.lang.annotation.Target; 24 | 25 | /** A set of workloads to assign to a specific node. */ 26 | @Retention(RUNTIME) 27 | @Target(ANNOTATION_TYPE) 28 | public @interface CCMWorkload { 29 | 30 | /** 31 | * The workloads to assign to a specific node. 32 | * 33 | * @return The workloads to assign to a specifc node. 34 | */ 35 | Workload[] value() default {}; 36 | } 37 | -------------------------------------------------------------------------------- /tests/src/main/java/com/datastax/oss/dsbulk/tests/simulacron/annotations/SimulacronFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.tests.simulacron.annotations; 17 | 18 | import static java.lang.annotation.ElementType.METHOD; 19 | import static java.lang.annotation.RetentionPolicy.RUNTIME; 20 | 21 | import java.lang.annotation.Retention; 22 | import java.lang.annotation.Target; 23 | 24 | /** 25 | * Annotation that marks a method as being a factory for {@link 26 | * com.datastax.oss.simulacron.server.BoundCluster Simulacron} instances. 27 | * 28 | *

Such methods must be static and their return type must be {@link 29 | * com.datastax.oss.simulacron.common.cluster.ClusterSpec}. 30 | */ 31 | @Retention(RUNTIME) 32 | @Target(METHOD) 33 | public @interface SimulacronFactory {} 34 | -------------------------------------------------------------------------------- /tests/src/main/resources/ssl/client.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDqTCCApGgAwIBAgIERLZiJzANBgkqhkiG9w0BAQsFADCBhDELMAkGA1UEBhMCVVMxEzARBgNV 3 | BAgTCkNhbGlmb3JuaWExFDASBgNVBAcTC1NhbnRhIENsYXJhMRYwFAYDVQQKEw1EYXRhU3RheCBJ 4 | bmMuMRowGAYDVQQLExFEcml2ZXJzIGFuZCBUb29sczEWMBQGA1UEAxMNRHJpdmVyIENsaWVudDAe 5 | Fw0xNTAzMTIwMTA4MjRaFw0xNTA2MTAwMTA4MjRaMIGEMQswCQYDVQQGEwJVUzETMBEGA1UECBMK 6 | Q2FsaWZvcm5pYTEUMBIGA1UEBxMLU2FudGEgQ2xhcmExFjAUBgNVBAoTDURhdGFTdGF4IEluYy4x 7 | GjAYBgNVBAsTEURyaXZlcnMgYW5kIFRvb2xzMRYwFAYDVQQDEw1Ecml2ZXIgQ2xpZW50MIIBIjAN 8 | BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAq0J0EoZQnOv2KRrvwA+1ZL9VZ3hDdQMwkDfitoGN 9 | B6upvMUZpf8W+ReQmaY6yacYJthHzsZTd3G97Bw81/3VNHQB9PnXGmbupMLVXeFXysSCs1nPEdJl 10 | TBbJXWHSh41AE4ejJaoCoTuigKGwI9lTbOOPDz/WMcio9nagsCJdsdG2+TxmR7RlyzEIANJ0wpnL 11 | JEIeJmRS2loLVuCU4lZ9hDLN57cP9jEVD4Hk2kJD4Exx7G9HQFH+/63H6XtEDZsJcYldR7yBNsGr 12 | pz9CupULCS1R40ePQEIlUXhM4ft/hsljQybLQvvfXNVTvk5WgY7LNaBJy6A/Tfg32SXEn3wUvwID 13 | AQABoyEwHzAdBgNVHQ4EFgQUt+JDOeziZzHNYTFU/FL9PhDGqSQwDQYJKoZIhvcNAQELBQADggEB 14 | ADOYpa1f9dPcVLq3RiMytajHo3YJ0AQqGRzVgngkeRFSdhyy/y+/8D0/V5s6QbNt/l6x3FxkoiTR 15 | 1Lptf96eylnS5AkGQTgogJP53cSNrqkDL0IyyvErSiATEXNpBKz6ivY+e5J1GLTfX9Ylu8limzIq 16 | Y6YBnr8fMLD6XWraxtzzkJ9NIPhhaz696rxqr8ix6uy0mgxR/7/jUglreimZkLW40/qiABgX7Evw 17 | UqpuJWmqNbQP9UXecx/UJ0hdxxxuxkZsoRoQwWYhkeT4aGCLJv/hjiNTfFAt23uHe0LVfW/HqykW 18 | KoEj8F08mJVe5ZfpjF974i5qO9PU9XxvLfLjNvo= 19 | -----END CERTIFICATE----- 20 | -------------------------------------------------------------------------------- /url/src/test/java/com/datastax/oss/dsbulk/url/StdinStdoutURLStreamHandlerProviderTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.url; 17 | 18 | import static org.assertj.core.api.Assertions.assertThat; 19 | 20 | import org.junit.jupiter.api.Test; 21 | 22 | class StdinStdoutURLStreamHandlerProviderTest { 23 | 24 | @Test 25 | void should_handle_std_protocol() { 26 | StdinStdoutURLStreamHandlerProvider provider = new StdinStdoutURLStreamHandlerProvider(); 27 | assertThat(provider.maybeCreateURLStreamHandler("std", null)) 28 | .isNotNull() 29 | .containsInstanceOf(StdinStdoutURLStreamHandler.class); 30 | assertThat(provider.maybeCreateURLStreamHandler("STD", null)) 31 | .isNotNull() 32 | .containsInstanceOf(StdinStdoutURLStreamHandler.class); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /batcher/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 4.0.0 21 | 22 | dsbulk-parent 23 | com.datastax.oss 24 | 1.11.1-SNAPSHOT 25 | 26 | dsbulk-batcher 27 | pom 28 | DataStax Bulk Loader - Batcher 29 | Batcher API for the DataStax Bulk Loader. 30 | 31 | api 32 | reactor 33 | 34 | 35 | -------------------------------------------------------------------------------- /connectors/csv/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 25 | 26 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /connectors/json/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 25 | 26 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /distribution/src/conf/driver.conf: -------------------------------------------------------------------------------- 1 | #################################################################################################### 2 | # Java Driver configuration for DSBulk. 3 | # 4 | # See the Java Driver configuration reference for instructions on how to configure the driver 5 | # properly: 6 | # https://docs.datastax.com/en/developer/java-driver/latest/ 7 | # https://docs.datastax.com/en/developer/java-driver-dse/latest/ 8 | # 9 | # This file is written in HOCON format; see 10 | # https://github.com/typesafehub/config/blob/master/HOCON.md 11 | # for more information on its syntax. 12 | # 13 | # This file is not meant as the main configuration file for DSBulk, but rather to be included from 14 | # the main configuration file. We recommend that this file be named driver.conf and placed in the 15 | # /conf directory, alongside with another configuration file for DSBulk itself, named 16 | # application.conf. Also, for this setup to work, application.conf should include driver.conf, for 17 | # example by using an import directive. For other ways to configure this tool, refer to DataStax 18 | # Bulk Loader online documentation: 19 | # https://docs.datastax.com/en/dsbulk/doc/dsbulk/dsbulkRef.html 20 | #################################################################################################### 21 | 22 | datastax-java-driver { 23 | # Example to set contact points: 24 | # basic.contact-points = [ "host1.com:9042" , "host2.com:9042" ] 25 | } 26 | -------------------------------------------------------------------------------- /executor/api/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 25 | 26 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /executor/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 4.0.0 21 | 22 | dsbulk-parent 23 | com.datastax.oss 24 | 1.11.1-SNAPSHOT 25 | 26 | dsbulk-executor 27 | pom 28 | DataStax Bulk Loader - Executor 29 | Bulk Executor for the DataStax Bulk Loader. 30 | 31 | api 32 | reactor 33 | 34 | 35 | -------------------------------------------------------------------------------- /workflow/commons/src/main/java/com/datastax/oss/dsbulk/workflow/commons/auth/BulkGssApiAuthProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.workflow.commons.auth; 17 | 18 | import com.datastax.dse.driver.api.core.auth.DseGssApiAuthProviderBase; 19 | import com.datastax.oss.driver.api.core.metadata.EndPoint; 20 | import edu.umd.cs.findbugs.annotations.NonNull; 21 | 22 | public class BulkGssApiAuthProvider extends DseGssApiAuthProviderBase { 23 | 24 | private final GssApiOptions options; 25 | 26 | public BulkGssApiAuthProvider(GssApiOptions options) { 27 | super(""); 28 | this.options = options; 29 | } 30 | 31 | @NonNull 32 | @Override 33 | protected GssApiOptions getOptions( 34 | @NonNull EndPoint endPoint, @NonNull String serverAuthenticator) { 35 | return options; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /url/src/main/java/com/datastax/oss/dsbulk/url/UncloseableInputStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.url; 17 | 18 | import java.io.FilterInputStream; 19 | import java.io.IOException; 20 | import java.io.InputStream; 21 | 22 | /** 23 | * An input stream that cannot be closed. 24 | * 25 | *

This is useful in rare situations where the underlying stream is being shared among consumers 26 | * and therefore should not be closed accidentally by one of them. 27 | */ 28 | @SuppressWarnings("WeakerAccess") 29 | public class UncloseableInputStream extends FilterInputStream { 30 | 31 | public UncloseableInputStream(InputStream in) { 32 | super(in); 33 | } 34 | 35 | @Override 36 | public void close() throws IOException { 37 | // do not forward the call to the delegate InputStream 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /workflow/commons/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 25 | 26 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /codecs/api/src/main/java/com/datastax/oss/dsbulk/codecs/api/ConvertingCodecProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.codecs.api; 17 | 18 | import com.datastax.oss.driver.api.core.type.DataType; 19 | import com.datastax.oss.driver.api.core.type.reflect.GenericType; 20 | import edu.umd.cs.findbugs.annotations.NonNull; 21 | import java.util.Optional; 22 | 23 | /** 24 | * A provider for {@link ConvertingCodec}s. Providers are discovered by {@link 25 | * ConvertingCodecFactory} using the Service Loader API. 26 | */ 27 | public interface ConvertingCodecProvider { 28 | 29 | @NonNull 30 | Optional> maybeProvide( 31 | @NonNull DataType cqlType, 32 | @NonNull GenericType externalJavaType, 33 | @NonNull ConvertingCodecFactory codecFactory, 34 | boolean rootCodec); 35 | } 36 | -------------------------------------------------------------------------------- /codecs/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 4.0.0 21 | 22 | dsbulk-parent 23 | com.datastax.oss 24 | 1.11.1-SNAPSHOT 25 | 26 | dsbulk-codecs 27 | pom 28 | DataStax Bulk Loader - Codecs 29 | Codecs for the DataStax Bulk Loader. 30 | 31 | api 32 | jdk 33 | text 34 | 35 | 36 | -------------------------------------------------------------------------------- /url/src/main/java/com/datastax/oss/dsbulk/url/UncloseableOutputStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.url; 17 | 18 | import java.io.FilterOutputStream; 19 | import java.io.IOException; 20 | import java.io.OutputStream; 21 | 22 | /** 23 | * An output stream that cannot be closed. 24 | * 25 | *

This is useful in rare situations where the underlying stream is being shared among consumers 26 | * and therefore should not be closed accidentally by one of them. 27 | */ 28 | @SuppressWarnings("WeakerAccess") 29 | public class UncloseableOutputStream extends FilterOutputStream { 30 | 31 | public UncloseableOutputStream(OutputStream out) { 32 | super(out); 33 | } 34 | 35 | @Override 36 | public void close() throws IOException { 37 | // do not forward the call to the delegate OutputStream 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /tests/src/main/java/com/datastax/oss/dsbulk/tests/logging/LogConfigurationResource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.tests.logging; 17 | 18 | import java.lang.annotation.ElementType; 19 | import java.lang.annotation.Retention; 20 | import java.lang.annotation.RetentionPolicy; 21 | import java.lang.annotation.Target; 22 | 23 | /** 24 | * The classpath resource that should be used to configure Logback. If this annotation is present, 25 | * {@link LogInterceptingExtension} will reset Logback's configuration and use this resource to 26 | * configure it. This can be useful in situations where a test is sensitive to Logback's 27 | * configuration. 28 | */ 29 | @Retention(RetentionPolicy.RUNTIME) 30 | @Target(ElementType.TYPE) 31 | public @interface LogConfigurationResource { 32 | 33 | String value() default "logback-test.xml"; 34 | } 35 | -------------------------------------------------------------------------------- /executor/reactor/src/test/java/com/datastax/oss/dsbulk/executor/reactor/DefaultReactorBulkExecutorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.executor.reactor; 17 | 18 | import com.datastax.oss.dsbulk.executor.api.AbstractBulkExecutorBuilder; 19 | import com.datastax.oss.dsbulk.executor.api.BulkExecutor; 20 | import com.datastax.oss.dsbulk.executor.api.NonContinuousBulkExecutorTestBase; 21 | 22 | public class DefaultReactorBulkExecutorTest extends NonContinuousBulkExecutorTestBase { 23 | 24 | @Override 25 | protected BulkExecutor newBulkExecutor(boolean failSafe) { 26 | AbstractBulkExecutorBuilder builder = 27 | DefaultReactorBulkExecutor.builder(session).withExecutionListener(listener); 28 | if (failSafe) { 29 | builder.failSafe(); 30 | } 31 | return builder.build(); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /workflow/api/src/main/java/com/datastax/oss/dsbulk/workflow/api/error/TooManyErrorsException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.workflow.api.error; 17 | 18 | /** 19 | * Thrown when the engine encounters too many errors. This exception triggers the operation 20 | * abortion. 21 | */ 22 | public class TooManyErrorsException extends RuntimeException { 23 | 24 | private final ErrorThreshold threshold; 25 | 26 | public TooManyErrorsException(ErrorThreshold threshold) { 27 | super(createErrorMessage(threshold)); 28 | this.threshold = threshold; 29 | } 30 | 31 | private static String createErrorMessage(ErrorThreshold threshold) { 32 | return "Too many errors, the maximum allowed is " + threshold.thresholdAsString() + "."; 33 | } 34 | 35 | public ErrorThreshold getThreshold() { 36 | return threshold; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /executor/reactor/src/main/java/com/datastax/oss/dsbulk/executor/reactor/ContinuousReactorBulkExecutorBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.executor.reactor; 17 | 18 | import com.datastax.oss.driver.api.core.CqlSession; 19 | import com.datastax.oss.dsbulk.executor.api.AbstractBulkExecutorBuilder; 20 | 21 | /** A builder for {@link ContinuousReactorBulkExecutor} instances. */ 22 | public class ContinuousReactorBulkExecutorBuilder 23 | extends AbstractBulkExecutorBuilder { 24 | 25 | final CqlSession cqlSession; 26 | 27 | ContinuousReactorBulkExecutorBuilder(CqlSession cqlSession) { 28 | super(cqlSession); 29 | this.cqlSession = cqlSession; 30 | } 31 | 32 | @Override 33 | public ContinuousReactorBulkExecutor build() { 34 | return new ContinuousReactorBulkExecutor(this); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /connectors/api/src/main/java/com/datastax/oss/dsbulk/connectors/api/RecordMetadata.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.connectors.api; 17 | 18 | import com.datastax.oss.driver.api.core.type.DataType; 19 | import com.datastax.oss.driver.api.core.type.reflect.GenericType; 20 | import edu.umd.cs.findbugs.annotations.NonNull; 21 | 22 | /** 23 | * Defines metadata applicable to a {@link Record record}, in particular which field types it 24 | * contains. 25 | */ 26 | public interface RecordMetadata { 27 | 28 | /** 29 | * Returns the type of the given field. 30 | * 31 | * @param field the field to get the type from. 32 | * @param cqlType the CQL type associated with the given field. 33 | * @return the type of the given field. 34 | */ 35 | @NonNull 36 | GenericType getFieldType(@NonNull Field field, @NonNull DataType cqlType); 37 | } 38 | -------------------------------------------------------------------------------- /codecs/jdk/src/test/java/com/datastax/oss/dsbulk/codecs/jdk/bool/BooleanToStringCodecTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.codecs.jdk.bool; 17 | 18 | import static com.datastax.oss.dsbulk.tests.assertions.TestAssertions.assertThat; 19 | 20 | import org.junit.jupiter.api.Test; 21 | 22 | class BooleanToStringCodecTest { 23 | 24 | private BooleanToStringCodec codec = new BooleanToStringCodec(); 25 | 26 | @Test 27 | void should_convert_when_valid_input() { 28 | assertThat(codec) 29 | .convertsFromExternal(true) 30 | .toInternal("true") 31 | .convertsFromExternal(false) 32 | .toInternal("false") 33 | .convertsFromExternal(null) 34 | .toInternal(null) 35 | .convertsFromInternal(null) 36 | .toExternal(null) 37 | .convertsFromInternal("") 38 | .toExternal(null); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /connectors/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 4.0.0 21 | 22 | dsbulk-parent 23 | com.datastax.oss 24 | 1.11.1-SNAPSHOT 25 | 26 | dsbulk-connectors 27 | pom 28 | DataStax Bulk Loader - Connectors 29 | Connectors for the DataStax Bulk Loader. 30 | 31 | api 32 | commons 33 | csv 34 | json 35 | 36 | 37 | -------------------------------------------------------------------------------- /codecs/api/src/main/java/com/datastax/oss/dsbulk/codecs/api/format/binary/HexBinaryFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.codecs.api.format.binary; 17 | 18 | import com.datastax.oss.driver.api.core.data.ByteUtils; 19 | import java.nio.ByteBuffer; 20 | 21 | public class HexBinaryFormat implements BinaryFormat { 22 | 23 | public static final HexBinaryFormat INSTANCE = new HexBinaryFormat(); 24 | 25 | private HexBinaryFormat() {} 26 | 27 | @Override 28 | public ByteBuffer parse(String s) { 29 | if (s == null) { 30 | return null; 31 | } 32 | // DAT-573: consider empty string as empty byte array 33 | if (s.isEmpty()) { 34 | return ByteBuffer.allocate(0); 35 | } 36 | return ByteUtils.fromHexString(s); 37 | } 38 | 39 | @Override 40 | public String format(ByteBuffer bb) { 41 | return ByteUtils.toHexString(bb); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /config/src/main/java/com/datastax/oss/dsbulk/config/model/FixedSettingsGroup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.config.model; 17 | 18 | import java.util.List; 19 | import java.util.Set; 20 | import java.util.TreeSet; 21 | 22 | /** A group of particular settings. */ 23 | class FixedSettingsGroup implements SettingsGroup { 24 | 25 | private final Set settings; 26 | 27 | FixedSettingsGroup(List fixedSettings) { 28 | settings = new TreeSet<>(new SettingsComparator(fixedSettings)); 29 | settings.addAll(fixedSettings); 30 | } 31 | 32 | @Override 33 | public boolean addSetting(String settingName) { 34 | // Always return false because we want other groups to have a chance to add this 35 | // setting as well. 36 | return false; 37 | } 38 | 39 | @Override 40 | public Set getSettings() { 41 | return settings; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /codecs/text/src/main/java/com/datastax/oss/dsbulk/codecs/text/string/StringToVectorCodec.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.codecs.text.string; 17 | 18 | import com.datastax.oss.driver.api.core.data.CqlVector; 19 | import com.datastax.oss.driver.internal.core.type.codec.VectorCodec; 20 | import java.util.List; 21 | 22 | public class StringToVectorCodec 23 | extends StringConvertingCodec> { 24 | 25 | public StringToVectorCodec(VectorCodec targetCodec, List nullStrings) { 26 | super(targetCodec, nullStrings); 27 | } 28 | 29 | @Override 30 | public CqlVector externalToInternal(String s) { 31 | return this.internalCodec.parse(s); 32 | } 33 | 34 | @Override 35 | public String internalToExternal(CqlVector cqlVector) { 36 | return this.internalCodec.format(cqlVector); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /workflow/api/src/main/java/com/datastax/oss/dsbulk/workflow/api/error/AbsoluteErrorThreshold.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.workflow.api.error; 17 | 18 | import edu.umd.cs.findbugs.annotations.NonNull; 19 | 20 | public class AbsoluteErrorThreshold implements ErrorThreshold { 21 | 22 | private final long maxErrors; 23 | 24 | AbsoluteErrorThreshold(long maxErrors) { 25 | if (maxErrors < 0) { 26 | throw new IllegalArgumentException("maxErrors must be >= 0"); 27 | } 28 | this.maxErrors = maxErrors; 29 | } 30 | 31 | @Override 32 | public boolean checkThresholdExceeded(long errorCount, @NonNull Number totalItems) { 33 | return errorCount > maxErrors; 34 | } 35 | 36 | @Override 37 | public String thresholdAsString() { 38 | return Long.toString(maxErrors); 39 | } 40 | 41 | public long getMaxErrors() { 42 | return maxErrors; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /codecs/text/src/main/java/com/datastax/oss/dsbulk/codecs/text/TextConversionContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.codecs.text; 17 | 18 | import com.datastax.oss.dsbulk.codecs.api.CommonConversionContext; 19 | import com.datastax.oss.dsbulk.codecs.text.json.JsonCodecUtils; 20 | import com.fasterxml.jackson.databind.ObjectMapper; 21 | import edu.umd.cs.findbugs.annotations.NonNull; 22 | import java.util.Objects; 23 | 24 | public class TextConversionContext extends CommonConversionContext { 25 | 26 | public static final String OBJECT_MAPPER = "OBJECT_MAPPER"; 27 | 28 | public TextConversionContext() { 29 | addAttribute(OBJECT_MAPPER, JsonCodecUtils.getObjectMapper()); 30 | } 31 | 32 | public TextConversionContext setObjectMapper(@NonNull ObjectMapper objectMapper) { 33 | addAttribute(OBJECT_MAPPER, Objects.requireNonNull(objectMapper)); 34 | return this; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /executor/api/src/main/java/com/datastax/oss/dsbulk/executor/api/BulkExecutorBuilderFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.executor.api; 17 | 18 | import com.datastax.oss.driver.api.core.CqlSession; 19 | import edu.umd.cs.findbugs.annotations.NonNull; 20 | 21 | /** A factory for {@link BulkExecutorBuilder} instances. */ 22 | public interface BulkExecutorBuilderFactory { 23 | 24 | /** 25 | * Creates a new {@link BulkExecutorBuilder} for the given session. 26 | * 27 | * @param session The session to create an executor for. 28 | * @param useContinuousPagingForReads whether the builder should create executors using continuous 29 | * paging for reads. 30 | * @return A newly-allocated bulk executor builder. 31 | */ 32 | @NonNull 33 | BulkExecutorBuilder create( 34 | @NonNull CqlSession session, boolean useContinuousPagingForReads); 35 | } 36 | -------------------------------------------------------------------------------- /executor/reactor/src/main/java/com/datastax/oss/dsbulk/executor/reactor/ReactorBulkExecutorBuilderFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.executor.reactor; 17 | 18 | import com.datastax.oss.driver.api.core.CqlSession; 19 | import com.datastax.oss.dsbulk.executor.api.BulkExecutorBuilder; 20 | import com.datastax.oss.dsbulk.executor.api.BulkExecutorBuilderFactory; 21 | import edu.umd.cs.findbugs.annotations.NonNull; 22 | 23 | public class ReactorBulkExecutorBuilderFactory implements BulkExecutorBuilderFactory { 24 | 25 | @Override 26 | @NonNull 27 | public BulkExecutorBuilder create( 28 | @NonNull CqlSession session, boolean useContinuousPagingForReads) { 29 | return useContinuousPagingForReads 30 | ? ContinuousReactorBulkExecutor.continuousPagingBuilder(session) 31 | : DefaultReactorBulkExecutor.builder(session); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /runner/src/main/java/com/datastax/oss/dsbulk/runner/cli/SectionHelpRequestException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.runner.cli; 17 | 18 | import edu.umd.cs.findbugs.annotations.NonNull; 19 | import edu.umd.cs.findbugs.annotations.Nullable; 20 | 21 | /** Simple exception indicating that the user wants the help for a particular section. */ 22 | public class SectionHelpRequestException extends Exception { 23 | 24 | private final String sectionName; 25 | @Nullable private final String connectorName; 26 | 27 | public SectionHelpRequestException(@NonNull String sectionName, @Nullable String connectorName) { 28 | this.sectionName = sectionName; 29 | this.connectorName = connectorName; 30 | } 31 | 32 | public String getSectionName() { 33 | return sectionName; 34 | } 35 | 36 | @Nullable 37 | public String getConnectorName() { 38 | return connectorName; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /workflow/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 4.0.0 21 | 22 | dsbulk-parent 23 | com.datastax.oss 24 | 1.11.1-SNAPSHOT 25 | 26 | dsbulk-workflow 27 | pom 28 | DataStax Bulk Loader - Workflow 29 | Workflows for the DataStax Bulk Loader. 30 | 31 | api 32 | commons 33 | load 34 | unload 35 | count 36 | 37 | 38 | -------------------------------------------------------------------------------- /codecs/text/src/main/java/com/datastax/oss/dsbulk/codecs/text/string/StringToStringCodec.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.codecs.text.string; 17 | 18 | import com.datastax.oss.driver.api.core.type.codec.TypeCodec; 19 | import java.util.List; 20 | 21 | public class StringToStringCodec extends StringConvertingCodec { 22 | 23 | public StringToStringCodec(TypeCodec innerCodec, List nullStrings) { 24 | super(innerCodec, nullStrings); 25 | } 26 | 27 | @Override 28 | public String externalToInternal(String s) { 29 | // DAT-297: do not convert empty strings to null so do not use isNullOrEmpty() here 30 | if (isNull(s)) { 31 | return null; 32 | } 33 | return s; 34 | } 35 | 36 | @Override 37 | public String internalToExternal(String value) { 38 | if (value == null) { 39 | return nullString(); 40 | } 41 | return value; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /codecs/text/src/main/java/com/datastax/oss/dsbulk/codecs/text/string/StringToDurationCodec.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.codecs.text.string; 17 | 18 | import com.datastax.oss.driver.api.core.data.CqlDuration; 19 | import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; 20 | import java.util.List; 21 | 22 | public class StringToDurationCodec extends StringConvertingCodec { 23 | 24 | public StringToDurationCodec(List nullStrings) { 25 | super(TypeCodecs.DURATION, nullStrings); 26 | } 27 | 28 | @Override 29 | public CqlDuration externalToInternal(String s) { 30 | if (isNullOrEmpty(s)) { 31 | return null; 32 | } 33 | return CqlDuration.from(s); 34 | } 35 | 36 | @Override 37 | public String internalToExternal(CqlDuration value) { 38 | if (value == null) { 39 | return nullString(); 40 | } 41 | return value.toString(); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /workflow/commons/src/main/java/com/datastax/oss/dsbulk/workflow/commons/settings/RowType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.workflow.commons.settings; 17 | 18 | public enum RowType { 19 | REGULAR { 20 | @Override 21 | public String singular() { 22 | return "row"; 23 | } 24 | 25 | @Override 26 | public String plural() { 27 | return "rows"; 28 | } 29 | }, 30 | VERTEX { 31 | @Override 32 | public String singular() { 33 | return "vertex"; 34 | } 35 | 36 | @Override 37 | public String plural() { 38 | return "vertices"; 39 | } 40 | }, 41 | EDGE { 42 | @Override 43 | public String singular() { 44 | return "edge"; 45 | } 46 | 47 | @Override 48 | public String plural() { 49 | return "edges"; 50 | } 51 | }; 52 | 53 | public abstract String singular(); 54 | 55 | public abstract String plural(); 56 | } 57 | -------------------------------------------------------------------------------- /url/src/test/java/com/datastax/oss/dsbulk/url/S3URLStreamHandlerProviderTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.url; 17 | 18 | import static org.assertj.core.api.Assertions.assertThat; 19 | import static org.mockito.Mockito.mock; 20 | import static org.mockito.Mockito.when; 21 | 22 | import com.typesafe.config.Config; 23 | import org.junit.jupiter.api.Test; 24 | 25 | class S3URLStreamHandlerProviderTest { 26 | 27 | @Test 28 | void should_handle_s3_protocol() { 29 | Config config = mock(Config.class); 30 | when(config.hasPath("dsbulk.s3.clientCacheSize")).thenReturn(true); 31 | when(config.getInt("dsbulk.s3.clientCacheSize")).thenReturn(25); 32 | 33 | S3URLStreamHandlerProvider provider = new S3URLStreamHandlerProvider(); 34 | 35 | assertThat(provider.maybeCreateURLStreamHandler("s3", config)) 36 | .isNotNull() 37 | .containsInstanceOf(S3URLStreamHandler.class); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /tests/src/main/resources/logback-template.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | %-5level [%thread] %logger{40} - %msg%n 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /mapping/src/test/java/com/datastax/oss/dsbulk/mapping/TypedCQLLiteralTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.mapping; 17 | 18 | import static org.assertj.core.api.Assertions.assertThat; 19 | 20 | import com.datastax.oss.driver.api.core.type.DataTypes; 21 | import org.junit.jupiter.api.Test; 22 | 23 | class TypedCQLLiteralTest { 24 | 25 | @Test 26 | void should_render() { 27 | TypedCQLLiteral literal = new TypedCQLLiteral("123", DataTypes.INT); 28 | assertThat(literal.render(CQLRenderMode.UNALIASED_SELECTOR)).isEqualTo("(int)123"); 29 | assertThat(literal.render(CQLRenderMode.ALIASED_SELECTOR)) 30 | .isEqualTo("(int)123 AS \"(int)123\""); 31 | assertThat(literal.render(CQLRenderMode.INTERNAL)).isEqualTo("(int)123"); 32 | assertThat(literal.render(CQLRenderMode.NAMED_ASSIGNMENT)).isEqualTo("123"); 33 | assertThat(literal.render(CQLRenderMode.POSITIONAL_ASSIGNMENT)).isEqualTo("123"); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /workflow/api/src/main/java/com/datastax/oss/dsbulk/workflow/api/utils/ConsoleUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.workflow.api.utils; 17 | 18 | public class ConsoleUtils { 19 | 20 | /** The console's line length, if could be detected, otherwise this will report 150. */ 21 | public static final int LINE_LENGTH = ConsoleUtils.getLineLength(); 22 | 23 | private static final String COLUMNS_ENV_NAME = "COLUMNS"; 24 | 25 | private static final int DEFAULT_LINE_LENGTH = 150; 26 | 27 | private static int getLineLength() { 28 | int columns = DEFAULT_LINE_LENGTH; 29 | String columnsStr = System.getenv(COLUMNS_ENV_NAME); 30 | if (columnsStr != null) { 31 | try { 32 | columns = Integer.parseInt(columnsStr); 33 | } catch (NumberFormatException ignored) { 34 | } 35 | if (PlatformUtils.isWindows()) { 36 | columns--; 37 | } 38 | } 39 | return columns; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /codecs/text/src/main/java/com/datastax/oss/dsbulk/codecs/text/json/JsonNodeToListCodec.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.codecs.text.json; 17 | 18 | import com.datastax.oss.driver.api.core.type.codec.TypeCodec; 19 | import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; 20 | import com.datastax.oss.dsbulk.codecs.api.ConvertingCodec; 21 | import com.fasterxml.jackson.databind.JsonNode; 22 | import com.fasterxml.jackson.databind.ObjectMapper; 23 | import java.util.ArrayList; 24 | import java.util.List; 25 | 26 | public class JsonNodeToListCodec extends JsonNodeToCollectionCodec> { 27 | 28 | public JsonNodeToListCodec( 29 | TypeCodec> collectionCodec, 30 | ConvertingCodec eltCodec, 31 | ObjectMapper objectMapper, 32 | List nullStrings) { 33 | super(collectionCodec, eltCodec, objectMapper, ArrayList::new, nullStrings, ImmutableList.of()); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /codecs/jdk/src/main/java/com/datastax/oss/dsbulk/codecs/jdk/bool/BooleanToStringCodec.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.codecs.jdk.bool; 17 | 18 | import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; 19 | import com.datastax.oss.dsbulk.codecs.api.ConvertingCodec; 20 | 21 | public class BooleanToStringCodec extends ConvertingCodec { 22 | 23 | public BooleanToStringCodec() { 24 | super(TypeCodecs.TEXT, Boolean.class); 25 | } 26 | 27 | @Override 28 | public Boolean internalToExternal(String value) { 29 | if (value == null || value.isEmpty()) { 30 | return null; 31 | } 32 | throw new UnsupportedOperationException( 33 | "This codec does not support converting from string to boolean"); 34 | } 35 | 36 | @Override 37 | public String externalToInternal(Boolean value) { 38 | if (value == null) { 39 | return null; 40 | } 41 | return value.toString(); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /workflow/api/src/main/java/com/datastax/oss/dsbulk/workflow/api/log/OperationDirectory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.workflow.api.log; 17 | 18 | import edu.umd.cs.findbugs.annotations.NonNull; 19 | import java.nio.file.Path; 20 | import java.nio.file.Paths; 21 | import java.util.Optional; 22 | 23 | public class OperationDirectory { 24 | 25 | private static final String OPERATION_DIRECTORY_KEY = 26 | "com.datastax.oss.dsbulk.OPERATION_DIRECTORY"; 27 | 28 | public static void setCurrentOperationDirectory(@NonNull Path operationDirectory) { 29 | System.setProperty(OPERATION_DIRECTORY_KEY, operationDirectory.toFile().getAbsolutePath()); 30 | } 31 | 32 | @NonNull 33 | public static Optional getCurrentOperationDirectory() { 34 | String path = System.getProperty(OPERATION_DIRECTORY_KEY); 35 | if (path == null) { 36 | return Optional.empty(); 37 | } 38 | return Optional.of(Paths.get(path)); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /codecs/api/src/main/java/com/datastax/oss/dsbulk/codecs/api/format/binary/BinaryFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.codecs.api.format.binary; 17 | 18 | import edu.umd.cs.findbugs.annotations.Nullable; 19 | import java.nio.ByteBuffer; 20 | 21 | /** An utility for parsing and formatting binary data. */ 22 | public interface BinaryFormat { 23 | 24 | /** 25 | * Parses the given string as a {@link ByteBuffer}. 26 | * 27 | * @param s the string to parse, may be {@code null}. 28 | * @return a {@link ByteBuffer} or {@code null} if the string was {@code null}. 29 | */ 30 | @Nullable 31 | ByteBuffer parse(@Nullable String s); 32 | 33 | /** 34 | * Formats the given {@link ByteBuffer} as a string. 35 | * 36 | * @param bytes the value to format, may be {@code null}. 37 | * @return the formatted value or {@code null} if the value was {@code null}. 38 | */ 39 | @Nullable 40 | String format(@Nullable ByteBuffer bytes); 41 | } 42 | -------------------------------------------------------------------------------- /workflow/commons/src/main/java/com/datastax/oss/dsbulk/workflow/commons/statement/UnmappableStatement.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.workflow.commons.statement; 17 | 18 | import com.datastax.oss.driver.api.core.cql.SimpleStatement; 19 | import com.datastax.oss.driver.shaded.guava.common.base.MoreObjects; 20 | import com.datastax.oss.dsbulk.connectors.api.Record; 21 | 22 | public class UnmappableStatement extends MappedSimpleStatement { 23 | 24 | private final Throwable error; 25 | 26 | public UnmappableStatement(Record record, Throwable error) { 27 | super(record, SimpleStatement.newInstance(error.getMessage())); 28 | this.error = error; 29 | } 30 | 31 | public Throwable getError() { 32 | return error; 33 | } 34 | 35 | @Override 36 | public String toString() { 37 | return MoreObjects.toStringHelper(this) 38 | .add("record", getRecord()) 39 | .add("error", error) 40 | .toString(); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /url/src/main/java/com/datastax/oss/dsbulk/url/StdinStdoutURLStreamHandlerProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.url; 17 | 18 | import com.typesafe.config.Config; 19 | import edu.umd.cs.findbugs.annotations.NonNull; 20 | import java.net.URLStreamHandler; 21 | import java.util.Optional; 22 | 23 | public class StdinStdoutURLStreamHandlerProvider implements URLStreamHandlerProvider { 24 | 25 | /** 26 | * The protocol for standard input and standard output URLs. The only supported URL with such 27 | * scheme is {@code std:/}. 28 | */ 29 | public static final String STANDARD_STREAM_PROTOCOL = "std"; 30 | 31 | @Override 32 | @NonNull 33 | public Optional maybeCreateURLStreamHandler( 34 | @NonNull String protocol, Config config) { 35 | if (STANDARD_STREAM_PROTOCOL.equalsIgnoreCase(protocol)) { 36 | return Optional.of(new StdinStdoutURLStreamHandler()); 37 | } 38 | return Optional.empty(); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /partitioner/src/it/java/com/datastax/oss/dsbulk/partitioner/M3PTokenPartitionerCCMIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.oss.dsbulk.partitioner; 17 | 18 | import com.datastax.oss.driver.api.core.CqlSession; 19 | import com.datastax.oss.dsbulk.tests.ccm.CCMCluster; 20 | import com.datastax.oss.dsbulk.tests.ccm.annotations.CCMConfig; 21 | import com.datastax.oss.dsbulk.tests.driver.annotations.SessionConfig; 22 | import com.datastax.oss.dsbulk.tests.driver.annotations.SessionConfig.UseKeyspaceMode; 23 | import org.junit.jupiter.api.Tag; 24 | 25 | @CCMConfig(numberOfNodes = 3) 26 | @Tag("long") 27 | class M3PTokenPartitionerCCMIT extends PartitionerCCMITBase { 28 | 29 | M3PTokenPartitionerCCMIT( 30 | CCMCluster ccm, 31 | @SessionConfig( 32 | useKeyspace = UseKeyspaceMode.NONE, 33 | settings = "basic.load-balancing-policy.slow-replica-avoidance=false") 34 | CqlSession session) { 35 | super(ccm, session, false); 36 | } 37 | } 38 | --------------------------------------------------------------------------------