├── .all-contributorsrc ├── .github ├── dependabot.yml ├── pull_request_template.md └── workflows │ ├── cdm-integrationtest.yml │ ├── docker-publish.yml │ ├── docker-push-sha-commit.yml │ ├── maven-publish.yml │ ├── maven.yml │ ├── snyk-cli-scan.yml │ └── snyk-pr-cleanup.yml ├── .gitignore ├── .snyk ├── .snyk.ignore.example ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE.md ├── PERF ├── cdm-v3.properties ├── cdm-v4.properties ├── cgpt.yaml ├── cql-iot.yaml ├── logs │ ├── scenario_20230523_162115_962.log │ ├── scenario_20230523_162126_056.log │ ├── scenario_20230523_162138_788.log │ ├── scenario_20230523_162151_747.log │ ├── scenario_20230523_162204_904.log │ ├── scenario_20230523_162859_122.log │ ├── scenario_20230523_162909_904.log │ └── scenario_20230523_164956_671.log ├── perf-iot.yaml └── testing.txt ├── README.md ├── RELEASE.md ├── SIT ├── Makefile ├── cdm-assert.sh ├── cdm.sh ├── common.sh ├── environment.sh ├── features │ ├── 01_constant_column │ │ ├── breakData.cql │ │ ├── cdm.fixData.assert │ │ ├── cdm.migrateData.assert │ │ ├── cdm.txt │ │ ├── cdm.validateData.assert │ │ ├── execute.sh │ │ ├── expected.cql │ │ ├── expected.out │ │ ├── fix.properties │ │ ├── migrate.properties │ │ └── setup.cql │ ├── 02_explode_map │ │ ├── breakData.cql │ │ ├── cdm.fixData.assert │ │ ├── cdm.migrateData.assert │ │ ├── cdm.txt │ │ ├── cdm.validateData.assert │ │ ├── execute.sh │ │ ├── expected.cql │ │ ├── expected.out │ │ ├── fix.properties │ │ ├── migrate.properties │ │ └── setup.cql │ ├── 03_codec │ │ ├── breakData.cql │ │ ├── cdm.fixData.assert │ │ ├── cdm.migrateData.assert │ │ ├── cdm.txt │ │ ├── cdm.validateData.assert │ │ ├── execute.sh │ │ ├── expected.cql │ │ ├── expected.out │ │ ├── fix.properties │ │ ├── migrate.properties │ │ └── setup.cql │ ├── 04_udt_mapper │ │ ├── breakData.cql │ │ ├── cdm.fixData.assert │ │ ├── cdm.migrateData.assert │ │ ├── cdm.txt │ │ ├── cdm.validateData.assert │ │ ├── execute.sh │ │ ├── expected.cql │ │ ├── expected.out │ │ ├── fix.properties │ │ ├── migrate.properties │ │ └── setup.cql │ ├── 05_guardrail │ │ ├── cdm.guardrailCheck.assert │ │ ├── cdm.txt │ │ ├── execute.sh │ │ ├── expected.cql │ │ ├── expected.out │ │ ├── migrate.properties │ │ └── setup.cql │ ├── 06_constant_column_remove │ │ ├── breakData.cql │ │ ├── cdm.fixData.assert │ │ ├── cdm.migrateData.assert │ │ ├── cdm.txt │ │ ├── cdm.validateData.assert │ │ ├── execute.sh │ │ ├── expected.cql │ │ ├── expected.out │ │ ├── fix.properties │ │ ├── migrate.properties │ │ └── setup.cql │ └── 07_constant_column_replace │ │ ├── breakData.cql │ │ ├── cdm.fixData.assert │ │ ├── cdm.migrateData.assert │ │ ├── cdm.txt │ │ ├── cdm.validateData.assert │ │ ├── execute.sh │ │ ├── expected.cql │ │ ├── expected.out │ │ ├── fix.properties │ │ ├── migrate.properties │ │ └── setup.cql ├── regression │ ├── 01_explode_map_with_constants │ │ ├── breakData.cql │ │ ├── cdm.fixData.assert │ │ ├── cdm.migrateData.assert │ │ ├── cdm.txt │ │ ├── cdm.validateData.assert │ │ ├── execute.sh │ │ ├── expected.cql │ │ ├── expected.out │ │ ├── fix.properties │ │ ├── migrate.properties │ │ └── setup.cql │ ├── 02_ColumnRenameWithConstantsAndExplode │ │ ├── breakData.cql │ │ ├── cdm.fixData.assert │ │ ├── cdm.migrateData.assert │ │ ├── cdm.txt │ │ ├── cdm.validateData.assert │ │ ├── execute.sh │ │ ├── expected.cql │ │ ├── expected.out │ │ ├── fix.properties │ │ ├── migrate.properties │ │ └── setup.cql │ ├── 03_performance │ │ ├── breakData.cql │ │ ├── cdm.fixData.assert │ │ ├── cdm.migrateData.assert │ │ ├── cdm.txt │ │ ├── cdm.validateData.assert │ │ ├── data.csv │ │ ├── data_break.csv │ │ ├── execute.sh │ │ ├── expected.cql │ │ ├── expected.out │ │ ├── fix.properties │ │ ├── migrate.properties │ │ └── setup.cql │ └── 04_null_ts_in_pk │ │ ├── cdm.sh │ │ ├── cdm.txt │ │ ├── execute.sh │ │ ├── expected.cql │ │ ├── expected.out │ │ ├── migrate.properties │ │ └── setup.cql ├── smoke │ ├── 00_test_harness │ │ ├── execute.sh │ │ ├── expected.cql │ │ ├── expected.out │ │ └── setup.cql │ ├── 01_basic_kvp │ │ ├── cdm.sh │ │ ├── cdm.txt │ │ ├── execute.sh │ │ ├── expected.cql │ │ ├── expected.out │ │ ├── migrate.properties │ │ └── setup.cql │ ├── 02_autocorrect_kvp │ │ ├── breakData.cql │ │ ├── cdm.txt │ │ ├── execute.sh │ │ ├── expected.cql │ │ ├── expected.out │ │ ├── migrate.properties │ │ └── setup.cql │ ├── 03_ttl_writetime │ │ ├── alternateCheckResults.sh │ │ ├── breakData.cql │ │ ├── cdm.fixData.assert │ │ ├── cdm.migrateData.assert │ │ ├── cdm.txt │ │ ├── cdm.validateData.assert │ │ ├── execute.sh │ │ ├── expected.cql │ │ ├── expected.out │ │ ├── fix.properties │ │ ├── migrate.properties │ │ ├── setup.cql │ │ └── t.out │ ├── 04_counters │ │ ├── breakData.cql │ │ ├── cdm.fixData.assert │ │ ├── cdm.fixForce.assert │ │ ├── cdm.migrateData.assert │ │ ├── cdm.txt │ │ ├── cdm.validateData.assert │ │ ├── execute.sh │ │ ├── expected.cql │ │ ├── expected.out │ │ ├── fix-force.properties │ │ ├── fix.properties │ │ ├── migrate.properties │ │ └── setup.cql │ ├── 05_reserved_keyword │ │ ├── cdm.sh │ │ ├── cdm.txt │ │ ├── execute.sh │ │ ├── expected.cql │ │ ├── expected.out │ │ ├── migrate.properties │ │ └── setup.cql │ └── 06_vector │ │ ├── cdm.migrateData.assert │ │ ├── cdm.txt │ │ ├── cdm.validateData.assert │ │ ├── execute.sh │ │ ├── expected.cql │ │ ├── expected.out │ │ ├── migrate.properties │ │ └── setup.cql └── test.sh ├── pom.xml ├── rat-excludes.txt ├── scripts ├── bump_minor_version └── get-latest-maven-version.sh ├── src ├── main │ ├── java │ │ └── com │ │ │ └── datastax │ │ │ └── cdm │ │ │ ├── cql │ │ │ ├── EnhancedSession.java │ │ │ ├── codec │ │ │ │ ├── ASCII_BLOBCodec.java │ │ │ │ ├── AbstractBaseCodec.java │ │ │ │ ├── BIGINT_BigIntegerCodec.java │ │ │ │ ├── BIGINT_StringCodec.java │ │ │ │ ├── BLOB_ASCIICodec.java │ │ │ │ ├── BLOB_TEXTCodec.java │ │ │ │ ├── BigInteger_BIGINTCodec.java │ │ │ │ ├── CodecFactory.java │ │ │ │ ├── Codecset.java │ │ │ │ ├── DECIMAL_StringCodec.java │ │ │ │ ├── DOUBLE_StringCodec.java │ │ │ │ ├── INT_StringCodec.java │ │ │ │ ├── TEXTFormat_InstantCodec.java │ │ │ │ ├── TEXTMillis_InstantCodec.java │ │ │ │ ├── TEXT_BLOBCodec.java │ │ │ │ ├── TEXT_BigDecimalCodec.java │ │ │ │ ├── TEXT_DoubleCodec.java │ │ │ │ ├── TEXT_IntegerCodec.java │ │ │ │ ├── TEXT_LongCodec.java │ │ │ │ ├── TIMESTAMP_StringFormatCodec.java │ │ │ │ └── TIMESTAMP_StringMillisCodec.java │ │ │ └── statement │ │ │ │ ├── BaseCdmStatement.java │ │ │ │ ├── OriginSelectByPKStatement.java │ │ │ │ ├── OriginSelectByPartitionRangeStatement.java │ │ │ │ ├── OriginSelectStatement.java │ │ │ │ ├── TargetInsertStatement.java │ │ │ │ ├── TargetSelectByPKStatement.java │ │ │ │ ├── TargetUpdateStatement.java │ │ │ │ ├── TargetUpsertRunDetailsStatement.java │ │ │ │ └── TargetUpsertStatement.java │ │ │ ├── data │ │ │ ├── AstraDevOpsClient.java │ │ │ ├── CqlConversion.java │ │ │ ├── CqlData.java │ │ │ ├── DataUtility.java │ │ │ ├── EnhancedPK.java │ │ │ ├── PKFactory.java │ │ │ └── Record.java │ │ │ ├── feature │ │ │ ├── AbstractFeature.java │ │ │ ├── ConstantColumns.java │ │ │ ├── ExplodeMap.java │ │ │ ├── ExtractJson.java │ │ │ ├── Feature.java │ │ │ ├── FeatureFactory.java │ │ │ ├── Featureset.java │ │ │ ├── Guardrail.java │ │ │ ├── OriginFilterCondition.java │ │ │ ├── TrackRun.java │ │ │ └── WritetimeTTL.java │ │ │ ├── job │ │ │ ├── AbstractJobSession.java │ │ │ ├── BaseJobSession.java │ │ │ ├── CDMMetricsAccumulator.java │ │ │ ├── CopyJobSession.java │ │ │ ├── CopyJobSessionFactory.java │ │ │ ├── CounterUnit.java │ │ │ ├── DiffJobSession.java │ │ │ ├── DiffJobSessionFactory.java │ │ │ ├── GuardrailCheckJobSession.java │ │ │ ├── GuardrailCheckJobSessionFactory.java │ │ │ ├── IJobSessionFactory.java │ │ │ ├── JobCounter.java │ │ │ ├── PartitionRange.java │ │ │ ├── RunNotStartedException.java │ │ │ └── SplitPartitions.java │ │ │ ├── properties │ │ │ ├── IPropertyHelper.java │ │ │ ├── KnownProperties.java │ │ │ └── PropertyHelper.java │ │ │ └── schema │ │ │ ├── BaseTable.java │ │ │ ├── CqlTable.java │ │ │ └── Table.java │ └── scala │ │ └── com │ │ └── datastax │ │ └── cdm │ │ └── job │ │ ├── BaseJob.scala │ │ ├── BasePartitionJob.scala │ │ ├── ConnectionDetails.scala │ │ ├── ConnectionFetcher.scala │ │ ├── DiffData.scala │ │ ├── GuardrailCheck.scala │ │ └── Migrate.scala ├── resources │ ├── cdm-detailed.properties │ ├── cdm.properties │ ├── log4j2.properties │ └── log4j2_docker.properties └── test │ └── java │ └── com │ └── datastax │ └── cdm │ ├── cql │ ├── CommonMocks.java │ ├── codec │ │ ├── ASCII_BLOBCodecTest.java │ │ ├── BIGINT_BigIntegerCodecTest.java │ │ ├── BIGINT_StringCodecTest.java │ │ ├── BLOB_ASCIICodecTest.java │ │ ├── BLOB_TextCodecTest.java │ │ ├── BigInteger_BIGINTCodecTest.java │ │ ├── CodecFactoryTest.java │ │ ├── CodecTestHelper.java │ │ ├── DATERANGETYPE_CodecTest.java │ │ ├── DECIMAL_StringCodecTest.java │ │ ├── DOUBLE_StringCodecTest.java │ │ ├── INT_StringCodecTest.java │ │ ├── LINESTRINGTYPE_CodecTest.java │ │ ├── POINTTYPE_CodecTest.java │ │ ├── POLYGONTYPE_CodecTest.java │ │ ├── TEXTFormat_InstantCodecTest.java │ │ ├── TEXTMillis_InstantCodecTest.java │ │ ├── TEXT_BLOBCodecTest.java │ │ ├── TEXT_BigDecimalCodecTest.java │ │ ├── TEXT_DoubleCodecTest.java │ │ ├── TEXT_IntegerCodecTest.java │ │ ├── TEXT_LongCodecTest.java │ │ ├── TIMESTAMP_StringFormatCodecTest.java │ │ └── TIMESTAMP_StringMillisCodecTest.java │ └── statement │ │ ├── BaseCdmStatementTest.java │ │ ├── Feature_CounterTest.java │ │ ├── OriginSelectByPKStatementTest.java │ │ ├── OriginSelectByPartitionRangeStatementTest.java │ │ ├── OriginSelectStatementTest.java │ │ ├── TargetInsertStatementTest.java │ │ ├── TargetSelectByPKStatementTest.java │ │ ├── TargetUpdateStatementTest.java │ │ ├── TargetUpsertRunDetailsStatementTest.java │ │ └── TargetUpsertStatementTest.java │ ├── data │ ├── AstraDevOpsClientTest.java │ ├── CqlConversionTest.java │ ├── DataUtilityTest.java │ ├── MockitoExtension.java │ └── RecordTest.java │ ├── feature │ ├── AbstractFeatureTest.java │ ├── ConstantColumnsTest.java │ ├── ExplodeMapTest.java │ ├── ExtractJsonTest.java │ ├── FeatureFactoryTest.java │ ├── GuardrailTest.java │ ├── OriginFilterConditionTest.java │ ├── TrackRunTest.java │ └── WritetimeTTLTest.java │ ├── job │ ├── ConnectionFetcherTest.java │ ├── JobCounterTest.java │ └── SplitPartitionsTest.java │ ├── properties │ ├── KnownPropertiesTest.java │ └── PropertyHelperTest.java │ ├── regression │ └── CQL_ColumnRenameWithConstantsAndExplode.java │ └── schema │ ├── BaseTableTest.java │ └── CqlTableTest.java └── test-backup └── feature └── UDTMapperTest.java /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # Set update schedule for GitHub Actions & Docker 2 | 3 | version: 2 4 | updates: 5 | - package-ecosystem: "github-actions" 6 | directory: "/" 7 | schedule: 8 | interval: "monthly" 9 | 10 | - package-ecosystem: "docker" 11 | directory: "/" 12 | schedule: 13 | interval: "monthly" 14 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | **What this PR does**: 4 | 5 | **Which issue(s) this PR fixes**: 6 | Fixes # 7 | 8 | **Checklist:** 9 | - [ ] Automated Tests added/updated 10 | - [ ] Documentation added/updated 11 | - [ ] CLA Signed: [DataStax CLA](https://cla.datastax.com/) 12 | -------------------------------------------------------------------------------- /.github/workflows/cdm-integrationtest.yml: -------------------------------------------------------------------------------- 1 | name: Build and test jar with integration tests 2 | on: 3 | workflow_dispatch: 4 | pull_request: 5 | push: 6 | branches: 7 | - main 8 | 9 | concurrency: 10 | group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' 11 | cancel-in-progress: true 12 | 13 | jobs: 14 | CDM-Integration-Test: 15 | strategy: 16 | matrix: 17 | jdk: [ 11, 17, 21 ] 18 | os: [ ubuntu-latest ] 19 | runs-on: ${{ matrix.os }} 20 | steps: 21 | - uses: actions/checkout@v4 22 | - name: Set up JDK 23 | uses: actions/setup-java@v4 24 | with: 25 | java-version: ${{ matrix.jdk }} 26 | distribution: 'temurin' 27 | cache: maven 28 | - name: Test SIT with cdm 29 | working-directory: SIT 30 | run: | 31 | echo "Testing SIT with cdm" 32 | make -f Makefile 33 | -------------------------------------------------------------------------------- /.github/workflows/docker-publish.yml: -------------------------------------------------------------------------------- 1 | name: Build and publish to docker repo 2 | 3 | on: workflow_dispatch 4 | 5 | jobs: 6 | build_and_publish: 7 | name: Build and publish docker image 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Check out source code 11 | uses: actions/checkout@v4 12 | # Setup metadata based on the commit/tag that will be used for tagging the image 13 | - name: Setup Docker meta 14 | id: meta 15 | uses: docker/metadata-action@v5 16 | with: 17 | images: | 18 | datastax/cassandra-data-migrator 19 | flavor: | 20 | latest=auto 21 | tags: | 22 | type=ref,event=branch 23 | type=semver,pattern={{version}} 24 | type=semver,pattern={{major}}.{{minor}}.x 25 | type=semver,pattern={{major}}.x 26 | - name: Set up Docker Buildx 27 | uses: docker/setup-buildx-action@v3 28 | - name: Login to DockerHub 29 | uses: docker/login-action@v3 30 | with: 31 | username: ${{ secrets.DOCKER_HUB_USERNAME }} 32 | password: ${{ secrets.DOCKER_HUB_PASSWORD }} 33 | - name: Build and push 34 | id: docker_build 35 | uses: docker/build-push-action@v6 36 | with: 37 | file: Dockerfile 38 | context: . 39 | push: true 40 | tags: ${{ steps.meta.outputs.tags }} 41 | platforms: linux/amd64 -------------------------------------------------------------------------------- /.github/workflows/docker-push-sha-commit.yml: -------------------------------------------------------------------------------- 1 | name: Build and publish commit release to docker repo 2 | 3 | on: workflow_dispatch 4 | 5 | jobs: 6 | build_and_publish: 7 | name: Build and publish docker image 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Check out source code 11 | uses: actions/checkout@v4 12 | # Setup metadata based on the commit/tag that will be used for tagging the image 13 | # Only build and publish a commit based tag 14 | - name: Setup Docker meta 15 | id: meta 16 | uses: docker/metadata-action@v5 17 | with: 18 | images: | 19 | datastax/cassandra-data-migrator 20 | flavor: | 21 | latest=false 22 | tags: | 23 | type=sha 24 | - name: Set up Docker Buildx 25 | uses: docker/setup-buildx-action@v3 26 | - name: Login to DockerHub 27 | uses: docker/login-action@v3 28 | with: 29 | username: ${{ secrets.DOCKER_HUB_USERNAME }} 30 | password: ${{ secrets.DOCKER_HUB_PASSWORD }} 31 | - name: Build and push 32 | id: docker_build 33 | uses: docker/build-push-action@v6 34 | with: 35 | file: Dockerfile 36 | context: . 37 | push: true 38 | tags: ${{ steps.meta.outputs.tags }} 39 | platforms: linux/amd64 40 | -------------------------------------------------------------------------------- /.github/workflows/maven-publish.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a package using Maven and then publish it to GitHub packages when a release is created 2 | # For more information see: https://github.com/actions/setup-java/blob/main/docs/advanced-usage.md#apache-maven-with-a-settings-path 3 | 4 | name: Maven Package 5 | 6 | on: 7 | workflow_dispatch: 8 | push: 9 | branches: [ main ] 10 | 11 | jobs: 12 | build: 13 | 14 | runs-on: ubuntu-latest 15 | permissions: 16 | contents: read 17 | packages: write 18 | 19 | steps: 20 | - uses: actions/checkout@v4 21 | - name: Set up JDK 22 | uses: actions/setup-java@v4 23 | with: 24 | java-version: '11' 25 | distribution: 'temurin' 26 | cache: maven 27 | server-id: github # Value of the distributionManagement/repository/id field of the pom.xml 28 | settings-path: ${{ github.workspace }} # location for the settings.xml file 29 | 30 | - name: Build with Maven 31 | run: mvn -B package --file pom.xml 32 | 33 | - name: Publish to GitHub Packages Apache Maven 34 | run: mvn deploy -s $GITHUB_WORKSPACE/settings.xml 35 | env: 36 | GITHUB_TOKEN: ${{ github.token }} 37 | 38 | clean: 39 | name: Cleanup Packages 40 | runs-on: ubuntu-latest 41 | needs: build 42 | steps: 43 | - name: Delete 4.x & 5.x Package Versions 44 | uses: actions/delete-package-versions@v5 45 | with: 46 | package-name: 'datastax.cdm.cassandra-data-migrator' 47 | package-type: 'maven' 48 | min-versions-to-keep: 7 49 | ignore-versions: '^4\.6\.1$' 50 | - name: Delete 3.x Package Versions 51 | uses: actions/delete-package-versions@v5 52 | with: 53 | package-name: 'datastax.astra.migrate.cassandra-data-migrator' 54 | package-type: 'maven' 55 | min-versions-to-keep: 6 56 | -------------------------------------------------------------------------------- /.github/workflows/maven.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven 3 | 4 | # This workflow uses actions that are not certified by GitHub. 5 | # They are provided by a third-party and are governed by 6 | # separate terms of service, privacy policy, and support 7 | # documentation. 8 | 9 | name: Java CI with Maven 10 | 11 | on: 12 | workflow_dispatch: 13 | pull_request: 14 | push: 15 | branches: 16 | - main 17 | 18 | concurrency: 19 | group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' 20 | cancel-in-progress: true 21 | 22 | jobs: 23 | build: 24 | strategy: 25 | matrix: 26 | jdk: [ 11, 17, 21 ] 27 | os: [ ubuntu-latest ] 28 | 29 | runs-on: ${{ matrix.os }} 30 | 31 | steps: 32 | - uses: actions/checkout@v4 33 | - name: Set up JDK 34 | uses: actions/setup-java@v4 35 | with: 36 | java-version: ${{ matrix.jdk }} 37 | distribution: 'temurin' 38 | cache: maven 39 | - name: Build with Maven 40 | run: mvn -B package --file pom.xml 41 | 42 | - name: bump minor version 43 | run: | 44 | scripts/bump_minor_version 45 | -------------------------------------------------------------------------------- /.github/workflows/snyk-cli-scan.yml: -------------------------------------------------------------------------------- 1 | # GitHub action CI 2 | # trigger by: 3 | # any push on any protected branch: main, v6.8, releases/** 4 | # any PR crteated against any protected branch: main, v6.8, releases/** 5 | 6 | name: 🔬 Snyk CLI scan 7 | 8 | on: 9 | push: 10 | branches: [ main ] 11 | pull_request: 12 | branches: [ main ] 13 | workflow_dispatch: 14 | 15 | concurrency: 16 | group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' 17 | #group: ${{ github.workflow }}-${{ github.ref }}-${{ github.job || github.run_id }} 18 | cancel-in-progress: true 19 | 20 | env: 21 | SNYK_SEVERITY_THRESHOLD_LEVEL: critical 22 | 23 | jobs: 24 | # this is a workaround for the issue that github actions does not support status check on workflow level 25 | skip-means-success: 26 | if: ${{ failure() }} 27 | runs-on: ubuntu-latest 28 | steps: 29 | - name: an always fail step 30 | run: | 31 | echo "if it reaches here, it means that some previous job(s) failed!" 32 | exit 123 33 | -------------------------------------------------------------------------------- /.github/workflows/snyk-pr-cleanup.yml: -------------------------------------------------------------------------------- 1 | # GitHub Action CI 2 | # Snyk clean-up when PR is merged/closed 3 | 4 | name: 🗑️ Snyk PR cleanup - merged/closed 5 | 6 | on: 7 | pull_request: 8 | types: 9 | - closed 10 | branches: 11 | - main 12 | workflow_dispatch: 13 | 14 | concurrency: 15 | group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' 16 | #group: ${{ github.workflow }}-${{ github.ref }}-${{ github.job || github.run_id }} 17 | cancel-in-progress: true 18 | 19 | jobs: 20 | # this is a workaround for the issue that github actions does not support status check on workflow level 21 | skip-means-success: 22 | if: ${{ failure() }} 23 | runs-on: ubuntu-latest 24 | steps: 25 | - name: an always fail step 26 | run: | 27 | echo "if it reaches here, it means that some previous job(s) failed!" 28 | exit 123 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | .bloop/* 3 | .metals/* 4 | .vscode/* 5 | dependency-reduced-pom.xml 6 | .idea/* 7 | cassandra-data-migrator.iml 8 | SIT/local 9 | SIT/**/output/* 10 | *.DS_Store 11 | .classpath 12 | .project 13 | .settings/* 14 | src/main/main.iml 15 | src/test/test.iml 16 | 17 | **/.claude/settings.local.json 18 | -------------------------------------------------------------------------------- /.snyk: -------------------------------------------------------------------------------- 1 | # .snyk 2 | # Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities. 3 | # See https://docs.snyk.io/scan-cloud-deployment/snyk-infrastructure-as-code/snyk-cli-for-infrastructure-as-code/iac-ignores-using-the-.snyk-policy-file for details. 4 | version: v1.22.2 5 | python: '3.7' 6 | patch: {} 7 | # ignores vulnerabilities until expiry date; change duration by modifying expiry date 8 | ignore: 9 | 10 | -------------------------------------------------------------------------------- /.snyk.ignore.example: -------------------------------------------------------------------------------- 1 | # .snyk.ignore.example 2 | # Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities. 3 | version: v1.22.2 4 | python: '3.7' 5 | patch: {} 6 | # ignores vulnerabilities until expiry date; change duration by modifying expiry date 7 | ignore: 8 | SNYK-PYTHON-URLLIB3-1533435: 9 | - '*': 10 | reason: state your ignore reason here 11 | expires: 2030-01-01T00:00:00.000Z 12 | created: 2022-03-21T13:19:22.196Z 13 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @datastax/cdm-core 2 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM eclipse-temurin:11-jammy 2 | 3 | # Download all migration dependencies 4 | RUN mkdir -p /assets/ && cd /assets && \ 5 | curl -OL https://downloads.datastax.com/dsbulk/dsbulk.tar.gz && \ 6 | tar -xzf ./dsbulk.tar.gz && \ 7 | rm ./dsbulk.tar.gz && \ 8 | mv /assets/dsbulk-* /assets/dsbulk && \ 9 | curl -OL https://downloads.datastax.com/enterprise/cqlsh-astra.tar.gz && \ 10 | tar -xzf ./cqlsh-astra.tar.gz && \ 11 | rm ./cqlsh-astra.tar.gz && \ 12 | curl -OL https://archive.apache.org/dist/spark/spark-3.5.5/spark-3.5.5-bin-hadoop3-scala2.13.tgz && \ 13 | tar -xzf ./spark-3.5.5-bin-hadoop3-scala2.13.tgz && \ 14 | rm ./spark-3.5.5-bin-hadoop3-scala2.13.tgz 15 | 16 | RUN apt-get update && apt-get install -y openssh-server vim python3 --no-install-recommends && \ 17 | rm -rf /var/lib/apt/lists/* && \ 18 | service ssh start 19 | 20 | # Copy CDM jar & template files 21 | ARG USER_HOME_DIR="/root" 22 | ENV MAVEN_HOME=/usr/share/maven 23 | ENV MAVEN_CONFIG="$USER_HOME_DIR/.m2" 24 | COPY ./src /assets/src 25 | COPY ./pom.xml /assets/pom.xml 26 | COPY ./src/resources/cdm.properties /assets/ 27 | COPY ./src/resources/cdm-detailed.properties /assets/ 28 | COPY ./scripts/get-latest-maven-version.sh ./get-latest-maven-version.sh 29 | 30 | RUN chmod +x ./get-latest-maven-version.sh && \ 31 | export MAVEN_VERSION=$(./get-latest-maven-version.sh) && \ 32 | export BASE_URL=https://dlcdn.apache.org/maven/maven-3/${MAVEN_VERSION}/binaries && \ 33 | mkdir -p /usr/share/maven /usr/share/maven/ref && \ 34 | curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz && \ 35 | tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 && \ 36 | rm -f /tmp/apache-maven.tar.gz && \ 37 | ln -s /usr/share/maven/bin/mvn /usr/bin/mvn && \ 38 | cd /assets && mvn -f ./pom.xml clean package && \ 39 | cp /assets/target/cassandra-data-migrator-*.jar /assets/ && \ 40 | rm -rf /assets/src && \ 41 | rm -rf /assets/target && \ 42 | rm -rf /assets/pom.xml && \ 43 | rm -rf "$MAVEN_HOME" && \ 44 | rm -rf "$USER_HOME_DIR/.m2" 45 | 46 | # Add all migration tools to path 47 | ENV PATH="${PATH}:/assets/dsbulk/bin/:/assets/cqlsh-astra/bin/:/assets/spark-3.5.5-bin-hadoop3-scala2.13/bin/" 48 | 49 | EXPOSE 22 50 | 51 | CMD ["/usr/sbin/sshd","-D"] 52 | -------------------------------------------------------------------------------- /PERF/logs/scenario_20230523_162115_962.log: -------------------------------------------------------------------------------- 1 | 2023-05-23 15:21:16,346 [main] INFO : Configured scenario log at logs/scenario_20230523_162115_962.log 2 | -------------------------------------------------------------------------------- /PERF/logs/scenario_20230523_162204_904.log: -------------------------------------------------------------------------------- 1 | 2023-05-23 15:22:05,249 [main] INFO : Configured scenario log at logs/scenario_20230523_162204_904.log 2 | 2023-05-23 15:22:05,253 [main] INFO : Running NoSQLBench Version 5.17.3-release 3 | 2023-05-23 15:22:05,262 [main] INFO : command-line: --copy cql-iot 4 | 2023-05-23 15:22:05,565 [main] INFO : client-hardware: {"processor-name":"11th Gen Intel(R) Core(TM) i7-11850H @ 2.50GHz","heap-max-GiB":"7.84","if-speeds":["10Gib"],"physical-cores":"3","max-frequency-ghz":"0.00","logical-cores":"6","sockets":"1","memory-GiB":"31.35"} 5 | 2023-05-23 15:22:05,692 [main] INFO : Copied internal resource '/activities/baselines/cql-iot.yaml' to 'cql-iot.yaml' 6 | -------------------------------------------------------------------------------- /PERF/logs/scenario_20230523_162859_122.log: -------------------------------------------------------------------------------- 1 | 2023-05-23 15:28:59,448 [main] INFO : Configured scenario log at logs/scenario_20230523_162859_122.log 2 | 2023-05-23 15:28:59,452 [main] INFO : Running NoSQLBench Version 5.17.3-release 3 | 2023-05-23 15:28:59,459 [main] INFO : command-line: --copy cql-iot 4 | 2023-05-23 15:28:59,761 [main] INFO : client-hardware: {"logical-cores":"6","sockets":"1","memory-GiB":"31.35","processor-name":"11th Gen Intel(R) Core(TM) i7-11850H @ 2.50GHz","heap-max-GiB":"7.84","if-speeds":["10Gib"],"physical-cores":"3","max-frequency-ghz":"0.00"} 5 | 2023-05-23 15:28:59,896 [main] ERROR: A file named cql-iot.yaml exists. Remove it first. 6 | 2023-05-23 15:28:59,899 [main] ERROR: for the full stack trace, run with --show-stacktraces 7 | -------------------------------------------------------------------------------- /PERF/logs/scenario_20230523_162909_904.log: -------------------------------------------------------------------------------- 1 | 2023-05-23 15:29:10,238 [main] INFO : Configured scenario log at logs/scenario_20230523_162909_904.log 2 | 2023-05-23 15:29:10,243 [main] INFO : Running NoSQLBench Version 5.17.3-release 3 | 2023-05-23 15:29:10,250 [main] INFO : command-line: --copy cql-iot 4 | 2023-05-23 15:29:10,541 [main] INFO : client-hardware: {"processor-name":"11th Gen Intel(R) Core(TM) i7-11850H @ 2.50GHz","memory-GiB":"31.35","sockets":"1","logical-cores":"6","max-frequency-ghz":"0.00","physical-cores":"3","if-speeds":["10Gib"],"heap-max-GiB":"7.84"} 5 | 2023-05-23 15:29:10,675 [main] INFO : Copied internal resource '/activities/baselines/cql-iot.yaml' to 'cql-iot.yaml' 6 | -------------------------------------------------------------------------------- /PERF/logs/scenario_20230523_164956_671.log: -------------------------------------------------------------------------------- 1 | 2023-05-23 15:49:57,045 [main] INFO : Configured scenario log at logs/scenario_20230523_164956_671.log 2 | 2023-05-23 15:49:57,049 [main] INFO : Running NoSQLBench Version 5.17.3-release 3 | 2023-05-23 15:49:57,056 [main] INFO : command-line: --help 4 | 2023-05-23 15:49:57,386 [main] INFO : client-hardware: {"logical-cores":"6","sockets":"1","memory-GiB":"31.35","processor-name":"11th Gen Intel(R) Core(TM) i7-11850H @ 2.50GHz","heap-max-GiB":"7.84","if-speeds":["10Gib"],"physical-cores":"3","max-frequency-ghz":"0.00"} 5 | -------------------------------------------------------------------------------- /SIT/Makefile: -------------------------------------------------------------------------------- 1 | 2 | all: setup test_smoke_cmd test_regression_cmd test_features_cmd env_teardown 3 | 4 | setup: build env_setup 5 | 6 | reset: build env_reset 7 | reset_hard: build env_teardown env_setup 8 | 9 | build: 10 | cd .. && mvn clean package 11 | 12 | test_smoke: reset test_smoke_cmd 13 | test_smoke_cmd: 14 | ./test.sh -p smoke 15 | 16 | test_regression: reset test_regression_cmd 17 | test_regression_cmd: 18 | ./test.sh -p regression 19 | 20 | test_features: reset test_features_cmd 21 | test_features_cmd: 22 | ./test.sh -p features 23 | 24 | test: test_smoke test_regression_cmd test_features_cmd 25 | 26 | # Local tests are not included in automated tests, but provide a means 27 | # to use the test harness to validate project-specific work 28 | test_local: reset test_local_cmd 29 | test_local_cmd: 30 | ./test.sh -p local 31 | 32 | env_setup: 33 | chmod -R 777 ./*.sh 34 | ./environment.sh -m setup -j ../target/cassandra-data-migrator*.jar 35 | env_reset: 36 | ./environment.sh -m reset -j ../target/cassandra-data-migrator*.jar 37 | env_validate: 38 | ./environment.sh -m validate 39 | env_teardown: 40 | ./environment.sh -m teardown 41 | 42 | -------------------------------------------------------------------------------- /SIT/common.sh: -------------------------------------------------------------------------------- 1 | export DOCKER_CASS=cdm-sit-cass 2 | export DOCKER_CDM=cdm-sit-cdm 3 | export CASS_USERNAME=cassandra 4 | export CASS_PASSWORD=cassandra 5 | export KEYSPACES="origin target" 6 | export CDM_DIRECTORY=/local 7 | export CDM_JARFILE=cassandra-data-migrator.jar 8 | 9 | _info() { 10 | echo "INFO $*" 11 | } 12 | 13 | _warn() { 14 | echo "WARN $*" 15 | } 16 | 17 | _error() { 18 | echo "ERROR $*" 19 | } 20 | 21 | _fatal() { 22 | echo "FATAL $*" 23 | exit 2 24 | } 25 | -------------------------------------------------------------------------------- /SIT/features/01_constant_column/breakData.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | DELETE FROM target.feature_constant_column WHERE key='key2' AND const1=1; 16 | UPDATE target.feature_constant_column SET value='value999' WHERE key='key3' AND const1=1; 17 | 18 | # This upsert to origin will update the writetime on origin to be newer than target 19 | INSERT INTO origin.feature_constant_column(key,value) VALUES ('key1','valueA'); 20 | INSERT INTO origin.feature_constant_column(key,value) VALUES ('key2','valueB'); 21 | INSERT INTO origin.feature_constant_column(key,value) VALUES ('key3','valueC'); 22 | 23 | SELECT * FROM target.feature_constant_column; 24 | 25 | -------------------------------------------------------------------------------- /SIT/features/01_constant_column/cdm.fixData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 3 2 | Mismatch Record Count: 1 3 | Corrected Mismatch Record Count: 1 4 | Missing Record Count: 1 5 | Corrected Missing Record Count: 1 6 | Valid Record Count: 1 7 | Skipped Record Count: 0 8 | Error Record Count: 0 9 | -------------------------------------------------------------------------------- /SIT/features/01_constant_column/cdm.migrateData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 3 2 | Skipped Record Count: 0 3 | Write Record Count: 3 4 | Error Record Count: 0 5 | -------------------------------------------------------------------------------- /SIT/features/01_constant_column/cdm.txt: -------------------------------------------------------------------------------- 1 | migrateData com.datastax.cdm.job.Migrate migrate.properties 2 | validateData com.datastax.cdm.job.DiffData migrate.properties 3 | fixData com.datastax.cdm.job.DiffData fix.properties 4 | -------------------------------------------------------------------------------- /SIT/features/01_constant_column/cdm.validateData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 3 2 | Mismatch Record Count: 0 3 | Corrected Mismatch Record Count: 0 4 | Missing Record Count: 0 5 | Corrected Missing Record Count: 0 6 | Valid Record Count: 3 7 | Skipped Record Count: 0 8 | Error Record Count: 0 9 | -------------------------------------------------------------------------------- /SIT/features/01_constant_column/execute.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | #!/bin/bash -e 16 | 17 | workingDir="$1" 18 | cd "$workingDir" 19 | 20 | /local/cdm.sh -f cdm.txt -s migrateData -d "$workingDir" > cdm.migrateData.out 2>cdm.migrateData.err 21 | /local/cdm-assert.sh -f cdm.migrateData.out -a cdm.migrateData.assert -d "$workingDir" 22 | 23 | /local/cdm.sh -f cdm.txt -s validateData -d "$workingDir" > cdm.validateData.out 2>cdm.validateData.err 24 | /local/cdm-assert.sh -f cdm.validateData.out -a cdm.validateData.assert -d "$workingDir" 25 | 26 | cqlsh -u $CASS_USERNAME -p $CASS_PASSWORD $CASS_CLUSTER -f $workingDir/breakData.cql > $workingDir/breakData.out 2> $workingDir/breakData.err 27 | 28 | /local/cdm.sh -f cdm.txt -s fixData -d "$workingDir" > cdm.fixData.out 2>cdm.fixData.err 29 | /local/cdm-assert.sh -f cdm.fixData.out -a cdm.fixData.assert -d "$workingDir" -------------------------------------------------------------------------------- /SIT/features/01_constant_column/expected.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | SELECT * FROM target.feature_constant_column; 16 | -------------------------------------------------------------------------------- /SIT/features/01_constant_column/expected.out: -------------------------------------------------------------------------------- 1 | 2 | const1 | key | const2 | value 3 | --------+------+--------+-------- 4 | 1 | key1 | 1234 | valueA 5 | 1 | key2 | 1234 | valueB 6 | 1 | key3 | 1234 | valueC 7 | 8 | (3 rows) 9 | -------------------------------------------------------------------------------- /SIT/features/01_constant_column/fix.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | spark.cdm.connect.origin.host cdm-sit-cass 16 | spark.cdm.connect.target.host cdm-sit-cass 17 | 18 | spark.cdm.schema.origin.keyspaceTable origin.feature_constant_column 19 | spark.cdm.schema.target.keyspaceTable target.feature_constant_column 20 | spark.cdm.perfops.numParts 1 21 | 22 | spark.cdm.feature.constantColumns.names const1,const2 23 | spark.cdm.feature.constantColumns.values 1,1234 24 | 25 | spark.cdm.autocorrect.missing true 26 | spark.cdm.autocorrect.mismatch true 27 | -------------------------------------------------------------------------------- /SIT/features/01_constant_column/migrate.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | spark.cdm.connect.origin.host cdm-sit-cass 16 | spark.cdm.connect.target.host cdm-sit-cass 17 | 18 | spark.cdm.schema.origin.keyspaceTable origin.feature_constant_column 19 | spark.cdm.schema.target.keyspaceTable target.feature_constant_column 20 | spark.cdm.perfops.numParts 1 21 | 22 | spark.cdm.feature.constantColumns.names const1,const2 23 | spark.cdm.feature.constantColumns.values 1,1234 24 | 25 | -------------------------------------------------------------------------------- /SIT/features/01_constant_column/setup.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | DROP TABLE IF EXISTS origin.feature_constant_column; 16 | CREATE TABLE origin.feature_constant_column(key text, value text, PRIMARY KEY (key)); 17 | INSERT INTO origin.feature_constant_column(key,value) VALUES ('key1','valueA'); 18 | INSERT INTO origin.feature_constant_column(key,value) VALUES ('key2','valueB'); 19 | INSERT INTO origin.feature_constant_column(key,value) VALUES ('key3','valueC'); 20 | 21 | DROP TABLE IF EXISTS target.feature_constant_column; 22 | CREATE TABLE target.feature_constant_column(const1 int, key text, value text, const2 int, PRIMARY KEY (const1, key)); 23 | -------------------------------------------------------------------------------- /SIT/features/02_explode_map/breakData.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | Unless required by applicable law or agreed to in writing, software 7 | distributed under the License is distributed on an "AS IS" BASIS, 8 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | See the License for the specific language governing permissions and 10 | limitations under the License. 11 | */ 12 | 13 | DELETE FROM target.feature_explode_map WHERE key='key2'; 14 | UPDATE target.feature_explode_map SET value='value999' WHERE key='key3' AND fruit='apples'; 15 | UPDATE target.feature_explode_map SET fruit_qty=999 WHERE key='key3' AND fruit='oranges'; 16 | DELETE FROM target.feature_explode_map WHERE key='key3' AND fruit='kiwi'; 17 | 18 | # This upsert to origin will update the writetime on origin to be newer than target 19 | INSERT INTO origin.feature_explode_map(key,value,fruits) VALUES ('key1','valueA', {'apples': 3, 'oranges': 5, 'bananas': 2, 'grapes': 11}); 20 | INSERT INTO origin.feature_explode_map(key,value,fruits) VALUES ('key2','valueB', {'apples': 4, 'oranges': 6, 'bananas': 3, 'pears': 7}); 21 | INSERT INTO origin.feature_explode_map(key,value,fruits) VALUES ('key3','valueC', {'apples': 5, 'oranges': 7, 'bananas': 4, 'kiwi': 42}); 22 | 23 | SELECT * FROM target.feature_explode_map; 24 | 25 | -------------------------------------------------------------------------------- /SIT/features/02_explode_map/cdm.fixData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 3 2 | Mismatch Record Count: 2 3 | Corrected Mismatch Record Count: 2 4 | Missing Record Count: 5 5 | Corrected Missing Record Count: 5 6 | Valid Record Count: 5 7 | Skipped Record Count: 0 8 | Error Record Count: 0 9 | -------------------------------------------------------------------------------- /SIT/features/02_explode_map/cdm.migrateData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 3 2 | Skipped Record Count: 0 3 | Write Record Count: 12 4 | Error Record Count: 0 5 | -------------------------------------------------------------------------------- /SIT/features/02_explode_map/cdm.txt: -------------------------------------------------------------------------------- 1 | migrateData com.datastax.cdm.job.Migrate migrate.properties 2 | validateData com.datastax.cdm.job.DiffData migrate.properties 3 | fixData com.datastax.cdm.job.DiffData fix.properties 4 | -------------------------------------------------------------------------------- /SIT/features/02_explode_map/cdm.validateData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 3 2 | Mismatch Record Count: 0 3 | Corrected Mismatch Record Count: 0 4 | Missing Record Count: 0 5 | Corrected Missing Record Count: 0 6 | Valid Record Count: 12 7 | Skipped Record Count: 0 8 | Error Record Count: 0 9 | -------------------------------------------------------------------------------- /SIT/features/02_explode_map/execute.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | #!/bin/bash -e 16 | 17 | workingDir="$1" 18 | cd "$workingDir" 19 | 20 | /local/cdm.sh -f cdm.txt -s migrateData -d "$workingDir" > cdm.migrateData.out 2>cdm.migrateData.err 21 | /local/cdm-assert.sh -f cdm.migrateData.out -a cdm.migrateData.assert -d "$workingDir" 22 | 23 | /local/cdm.sh -f cdm.txt -s validateData -d "$workingDir" > cdm.validateData.out 2>cdm.validateData.err 24 | /local/cdm-assert.sh -f cdm.validateData.out -a cdm.validateData.assert -d "$workingDir" 25 | 26 | cqlsh -u $CASS_USERNAME -p $CASS_PASSWORD $CASS_CLUSTER -f $workingDir/breakData.cql > $workingDir/breakData.out 2> $workingDir/breakData.err 27 | 28 | /local/cdm.sh -f cdm.txt -s fixData -d "$workingDir" > cdm.fixData.out 2>cdm.fixData.err 29 | /local/cdm-assert.sh -f cdm.fixData.out -a cdm.fixData.assert -d "$workingDir" 30 | -------------------------------------------------------------------------------- /SIT/features/02_explode_map/expected.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | Unless required by applicable law or agreed to in writing, software 7 | distributed under the License is distributed on an "AS IS" BASIS, 8 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | See the License for the specific language governing permissions and 10 | limitations under the License. 11 | */ 12 | 13 | SELECT * FROM target.feature_explode_map; 14 | -------------------------------------------------------------------------------- /SIT/features/02_explode_map/expected.out: -------------------------------------------------------------------------------- 1 | 2 | key | fruit | fruit_qty | value 3 | ------+---------+-----------+-------- 4 | key1 | apples | 3 | valueA 5 | key1 | bananas | 2 | valueA 6 | key1 | grapes | 11 | valueA 7 | key1 | oranges | 5 | valueA 8 | key3 | apples | 5 | valueC 9 | key3 | bananas | 4 | valueC 10 | key3 | kiwi | 42 | valueC 11 | key3 | oranges | 7 | valueC 12 | key2 | apples | 4 | valueB 13 | key2 | bananas | 3 | valueB 14 | key2 | oranges | 6 | valueB 15 | key2 | pears | 7 | valueB 16 | 17 | (12 rows) 18 | -------------------------------------------------------------------------------- /SIT/features/02_explode_map/fix.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | spark.cdm.connect.origin.host cdm-sit-cass 16 | spark.cdm.connect.target.host cdm-sit-cass 17 | 18 | spark.cdm.schema.origin.keyspaceTable origin.feature_explode_map 19 | spark.cdm.schema.target.keyspaceTable target.feature_explode_map 20 | spark.cdm.perfops.numParts 1 21 | 22 | spark.cdm.feature.explodeMap.origin.name fruits 23 | spark.cdm.feature.explodeMap.target.name.key fruit 24 | spark.cdm.feature.explodeMap.target.name.value fruit_qty 25 | 26 | spark.cdm.autocorrect.missing true 27 | spark.cdm.autocorrect.mismatch true 28 | 29 | -------------------------------------------------------------------------------- /SIT/features/02_explode_map/migrate.properties: -------------------------------------------------------------------------------- 1 | # 2 | # http://www.apache.org/licenses/LICENSE-2.0 3 | # 4 | # Unless required by applicable law or agreed to in writing, software 5 | # distributed under the License is distributed on an "AS IS" BASIS, 6 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 7 | # See the License for the specific language governing permissions and 8 | # limitations under the License. 9 | # 10 | 11 | spark.cdm.connect.origin.host cdm-sit-cass 12 | spark.cdm.connect.target.host cdm-sit-cass 13 | 14 | spark.cdm.schema.origin.keyspaceTable origin.feature_explode_map 15 | spark.cdm.schema.target.keyspaceTable target.feature_explode_map 16 | spark.cdm.perfops.numParts 1 17 | 18 | spark.cdm.feature.explodeMap.origin.name fruits 19 | spark.cdm.feature.explodeMap.target.name.key fruit 20 | spark.cdm.feature.explodeMap.target.name.value fruit_qty 21 | 22 | -------------------------------------------------------------------------------- /SIT/features/02_explode_map/setup.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | Unless required by applicable law or agreed to in writing, software 7 | distributed under the License is distributed on an "AS IS" BASIS, 8 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | See the License for the specific language governing permissions and 10 | limitations under the License. 11 | */ 12 | 13 | DROP TABLE IF EXISTS origin.feature_explode_map; 14 | CREATE TABLE origin.feature_explode_map(key text, value text, fruits map, PRIMARY KEY (key)); 15 | INSERT INTO origin.feature_explode_map(key,value,fruits) VALUES ('key1','valueA', {'apples': 3, 'oranges': 5, 'bananas': 2, 'grapes': 11}); 16 | INSERT INTO origin.feature_explode_map(key,value,fruits) VALUES ('key2','valueB', {'apples': 4, 'oranges': 6, 'bananas': 3, 'pears': 7}); 17 | INSERT INTO origin.feature_explode_map(key,value,fruits) VALUES ('key3','valueC', {'apples': 5, 'oranges': 7, 'bananas': 4, 'kiwi': 42}); 18 | 19 | DROP TABLE IF EXISTS target.feature_explode_map; 20 | CREATE TABLE target.feature_explode_map(key text, fruit text, value text, fruit_qty int, PRIMARY KEY ((key), fruit)); 21 | -------------------------------------------------------------------------------- /SIT/features/03_codec/breakData.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | Unless required by applicable law or agreed to in writing, software 7 | distributed under the License is distributed on an "AS IS" BASIS, 8 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | See the License for the specific language governing permissions and 10 | limitations under the License. 11 | */ 12 | 13 | 14 | DELETE FROM target.codec WHERE key='key2'; 15 | UPDATE target.codec SET 16 | val_int=654321, 17 | val_bigint=314159200000000, 18 | val_timestamp='2024-04-16 10:30:00+0000', 19 | val_decimal=999.1234 20 | WHERE key='key3'; 21 | 22 | # This upsert to origin will update the writetime on origin to be newer than target 23 | INSERT INTO origin.codec(key,val_int,val_bigint,val_timestamp,val_decimal,val_double) 24 | VALUES ('key1','1234' ,'9223372036854775807','040616110000', '3.14', '21474836470.7'); 25 | INSERT INTO origin.codec(key,val_int,val_bigint,val_timestamp,val_decimal,val_double) 26 | VALUES ('key2','12345' ,'2147483648' ,'990616110000', '4.14', '21474836470.7'); 27 | INSERT INTO origin.codec(key,val_int,val_bigint,val_timestamp,val_decimal,val_double) 28 | VALUES ('key3','123456','3141592653589793' ,'990616110000', '5.14', '21474836470.7'); 29 | 30 | SELECT * FROM target.codec; 31 | -------------------------------------------------------------------------------- /SIT/features/03_codec/cdm.fixData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 3 2 | Mismatch Record Count: 1 3 | Corrected Mismatch Record Count: 1 4 | Missing Record Count: 1 5 | Corrected Missing Record Count: 1 6 | Valid Record Count: 1 7 | Skipped Record Count: 0 8 | Error Record Count: 0 9 | -------------------------------------------------------------------------------- /SIT/features/03_codec/cdm.migrateData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 3 2 | Skipped Record Count: 0 3 | Write Record Count: 3 4 | Error Record Count: 0 5 | -------------------------------------------------------------------------------- /SIT/features/03_codec/cdm.txt: -------------------------------------------------------------------------------- 1 | migrateData com.datastax.cdm.job.Migrate migrate.properties 2 | validateData com.datastax.cdm.job.DiffData migrate.properties 3 | fixData com.datastax.cdm.job.DiffData fix.properties 4 | -------------------------------------------------------------------------------- /SIT/features/03_codec/cdm.validateData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 3 2 | Mismatch Record Count: 0 3 | Corrected Mismatch Record Count: 0 4 | Missing Record Count: 0 5 | Corrected Missing Record Count: 0 6 | Valid Record Count: 3 7 | Skipped Record Count: 0 8 | Error Record Count: 0 9 | -------------------------------------------------------------------------------- /SIT/features/03_codec/execute.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | #!/bin/bash -e 16 | 17 | workingDir="$1" 18 | cd "$workingDir" 19 | 20 | /local/cdm.sh -f cdm.txt -s migrateData -d "$workingDir" > cdm.migrateData.out 2>cdm.migrateData.err 21 | /local/cdm-assert.sh -f cdm.migrateData.out -a cdm.migrateData.assert -d "$workingDir" 22 | 23 | /local/cdm.sh -f cdm.txt -s validateData -d "$workingDir" > cdm.validateData.out 2>cdm.validateData.err 24 | /local/cdm-assert.sh -f cdm.validateData.out -a cdm.validateData.assert -d "$workingDir" 25 | 26 | cqlsh -u $CASS_USERNAME -p $CASS_PASSWORD $CASS_CLUSTER -f $workingDir/breakData.cql > $workingDir/breakData.out 2> $workingDir/breakData.err 27 | 28 | /local/cdm.sh -f cdm.txt -s fixData -d "$workingDir" > cdm.fixData.out 2>cdm.fixData.err 29 | /local/cdm-assert.sh -f cdm.fixData.out -a cdm.fixData.assert -d "$workingDir" 30 | -------------------------------------------------------------------------------- /SIT/features/03_codec/expected.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | Unless required by applicable law or agreed to in writing, software 7 | distributed under the License is distributed on an "AS IS" BASIS, 8 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | See the License for the specific language governing permissions and 10 | limitations under the License. 11 | */ 12 | 13 | SELECT * FROM target.codec; 14 | -------------------------------------------------------------------------------- /SIT/features/03_codec/expected.out: -------------------------------------------------------------------------------- 1 | 2 | key | val_bigint | val_decimal | val_double | val_int | val_timestamp 3 | ------+---------------------+-------------+------------+---------+--------------------------------- 4 | key1 | 9223372036854775807 | 3.14 | 2.1475e+10 | 1234 | 2004-06-16 11:00:00.000000+0000 5 | key3 | 3141592653589793 | 5.14 | 2.1475e+10 | 123456 | 2099-06-16 11:00:00.000000+0000 6 | key2 | 2147483648 | 4.14 | 2.1475e+10 | 12345 | 2099-06-16 11:00:00.000000+0000 7 | 8 | (3 rows) 9 | -------------------------------------------------------------------------------- /SIT/features/03_codec/fix.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | spark.cdm.connect.origin.host cdm-sit-cass 16 | spark.cdm.connect.target.host cdm-sit-cass 17 | 18 | spark.cdm.schema.origin.keyspaceTable origin.codec 19 | spark.cdm.schema.target.keyspaceTable target.codec 20 | spark.cdm.perfops.numParts 1 21 | 22 | spark.cdm.transform.codecs INT_STRING,DOUBLE_STRING,BIGINT_STRING,DECIMAL_STRING,TIMESTAMP_STRING_FORMAT 23 | spark.cdm.transform.codecs.timestamp.string.format yyMMddHHmmss 24 | 25 | spark.cdm.autocorrect.missing true 26 | spark.cdm.autocorrect.mismatch true 27 | 28 | -------------------------------------------------------------------------------- /SIT/features/03_codec/migrate.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | spark.cdm.connect.origin.host cdm-sit-cass 16 | spark.cdm.connect.target.host cdm-sit-cass 17 | 18 | spark.cdm.schema.origin.keyspaceTable origin.codec 19 | spark.cdm.schema.target.keyspaceTable target.codec 20 | spark.cdm.perfops.numParts 1 21 | 22 | spark.cdm.transform.codecs INT_STRING,DOUBLE_STRING,BIGINT_STRING,DECIMAL_STRING,TIMESTAMP_STRING_FORMAT 23 | spark.cdm.transform.codecs.timestamp.string.format yyMMddHHmmss 24 | 25 | -------------------------------------------------------------------------------- /SIT/features/03_codec/setup.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | Unless required by applicable law or agreed to in writing, software 7 | distributed under the License is distributed on an "AS IS" BASIS, 8 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | See the License for the specific language governing permissions and 10 | limitations under the License. 11 | */ 12 | 13 | DROP TABLE IF EXISTS origin.codec; 14 | CREATE TABLE origin.codec(key text, val_int text, val_bigint text, val_timestamp text, val_decimal text, val_double text, PRIMARY KEY (key)); 15 | INSERT INTO origin.codec(key,val_int,val_bigint,val_timestamp,val_decimal,val_double) 16 | VALUES ('key1','1234' ,'9223372036854775807','040616110000', '3.14', '21474836470.7'); 17 | INSERT INTO origin.codec(key,val_int,val_bigint,val_timestamp,val_decimal,val_double) 18 | VALUES ('key2','12345' ,'2147483648' ,'990616110000', '4.14', '21474836470.7'); 19 | INSERT INTO origin.codec(key,val_int,val_bigint,val_timestamp,val_decimal,val_double) 20 | VALUES ('key3','123456','3141592653589793' ,'990616110000', '5.14', '21474836470.7'); 21 | SELECT * FROM origin.codec; 22 | 23 | DROP TABLE IF EXISTS target.codec; 24 | CREATE TABLE target.codec(key text, val_int int, val_bigint bigint, val_timestamp timestamp, val_decimal decimal, val_double double, PRIMARY KEY (key)); 25 | -------------------------------------------------------------------------------- /SIT/features/04_udt_mapper/breakData.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | Unless required by applicable law or agreed to in writing, software 7 | distributed under the License is distributed on an "AS IS" BASIS, 8 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | See the License for the specific language governing permissions and 10 | limitations under the License. 11 | */ 12 | 13 | DELETE FROM target.udt_mapper WHERE key='key2'; 14 | UPDATE target.udt_mapper SET 15 | val_udt = {text_value: 'value9990', long_value: 9990000000, double_value: 999.00} 16 | ,val_list = [ {text_value: 'value9991', long_value: 9990000001, double_value: 999.01} 17 | ,{text_value: 'value9992', long_value: 9990000002, double_value: 999.02}] 18 | ,val_set = { {text_value: 'value9993', long_value: 9990000003, double_value: 999.03} 19 | ,{text_value: 'value9994', long_value: 9990000004, double_value: 999.04}} 20 | ,val_map = {'map_key1': {text_value: 'value9995', long_value: 9990000005, double_value: 999.05} 21 | ,'map_key2': {text_value: 'value9996', long_value: 9990000006, double_value: 999.06}} 22 | WHERE key='key3'; 23 | SELECT * FROM target.udt_mapper; 24 | -------------------------------------------------------------------------------- /SIT/features/04_udt_mapper/cdm.fixData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 3 2 | Mismatch Record Count: 1 3 | Corrected Mismatch Record Count: 1 4 | Missing Record Count: 1 5 | Corrected Missing Record Count: 1 6 | Valid Record Count: 1 7 | Skipped Record Count: 0 8 | Error Record Count: 0 9 | -------------------------------------------------------------------------------- /SIT/features/04_udt_mapper/cdm.migrateData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 3 2 | Skipped Record Count: 0 3 | Write Record Count: 3 4 | Error Record Count: 0 5 | -------------------------------------------------------------------------------- /SIT/features/04_udt_mapper/cdm.txt: -------------------------------------------------------------------------------- 1 | migrateData com.datastax.cdm.job.Migrate migrate.properties 2 | validateData com.datastax.cdm.job.DiffData migrate.properties 3 | fixData com.datastax.cdm.job.DiffData fix.properties 4 | -------------------------------------------------------------------------------- /SIT/features/04_udt_mapper/cdm.validateData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 3 2 | Mismatch Record Count: 0 3 | Corrected Mismatch Record Count: 0 4 | Missing Record Count: 0 5 | Corrected Missing Record Count: 0 6 | Valid Record Count: 3 7 | Skipped Record Count: 0 8 | Error Record Count: 0 9 | -------------------------------------------------------------------------------- /SIT/features/04_udt_mapper/execute.sh: -------------------------------------------------------------------------------- 1 | # 2 | # http://www.apache.org/licenses/LICENSE-2.0 3 | # 4 | # Unless required by applicable law or agreed to in writing, software 5 | # distributed under the License is distributed on an "AS IS" BASIS, 6 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 7 | # See the License for the specific language governing permissions and 8 | # limitations under the License. 9 | # 10 | 11 | #!/bin/bash -e 12 | 13 | workingDir="$1" 14 | cd "$workingDir" 15 | 16 | /local/cdm.sh -f cdm.txt -s migrateData -d "$workingDir" > cdm.migrateData.out 2>cdm.migrateData.err 17 | /local/cdm-assert.sh -f cdm.migrateData.out -a cdm.migrateData.assert -d "$workingDir" 18 | 19 | /local/cdm.sh -f cdm.txt -s validateData -d "$workingDir" > cdm.validateData.out 2>cdm.validateData.err 20 | /local/cdm-assert.sh -f cdm.validateData.out -a cdm.validateData.assert -d "$workingDir" 21 | 22 | cqlsh -u $CASS_USERNAME -p $CASS_PASSWORD $CASS_CLUSTER -f $workingDir/breakData.cql > $workingDir/breakData.out 2> $workingDir/breakData.err 23 | 24 | /local/cdm.sh -f cdm.txt -s fixData -d "$workingDir" > cdm.fixData.out 2>cdm.fixData.err 25 | /local/cdm-assert.sh -f cdm.fixData.out -a cdm.fixData.assert -d "$workingDir" 26 | -------------------------------------------------------------------------------- /SIT/features/04_udt_mapper/expected.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | Unless required by applicable law or agreed to in writing, software 7 | distributed under the License is distributed on an "AS IS" BASIS, 8 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | See the License for the specific language governing permissions and 10 | limitations under the License. 11 | */ 12 | 13 | SELECT * FROM target.udt_mapper; 14 | -------------------------------------------------------------------------------- /SIT/features/04_udt_mapper/fix.properties: -------------------------------------------------------------------------------- 1 | # 2 | # http://www.apache.org/licenses/LICENSE-2.0 3 | # 4 | # Unless required by applicable law or agreed to in writing, software 5 | # distributed under the License is distributed on an "AS IS" BASIS, 6 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 7 | # See the License for the specific language governing permissions and 8 | # limitations under the License. 9 | # 10 | 11 | spark.cdm.connect.origin.host cdm-sit-cass 12 | spark.cdm.connect.target.host cdm-sit-cass 13 | 14 | spark.cdm.schema.origin.keyspaceTable origin.udt_mapper 15 | spark.cdm.schema.target.keyspaceTable target.udt_mapper 16 | spark.cdm.perfops.numParts 1 17 | 18 | spark.cdm.autocorrect.missing true 19 | spark.cdm.autocorrect.mismatch true 20 | 21 | -------------------------------------------------------------------------------- /SIT/features/04_udt_mapper/migrate.properties: -------------------------------------------------------------------------------- 1 | # 2 | # http://www.apache.org/licenses/LICENSE-2.0 3 | # 4 | # Unless required by applicable law or agreed to in writing, software 5 | # distributed under the License is distributed on an "AS IS" BASIS, 6 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 7 | # See the License for the specific language governing permissions and 8 | # limitations under the License. 9 | # 10 | 11 | spark.cdm.connect.origin.host cdm-sit-cass 12 | spark.cdm.connect.target.host cdm-sit-cass 13 | 14 | spark.cdm.schema.origin.keyspaceTable origin.udt_mapper 15 | spark.cdm.schema.target.keyspaceTable target.udt_mapper 16 | spark.cdm.perfops.numParts 1 17 | -------------------------------------------------------------------------------- /SIT/features/05_guardrail/cdm.guardrailCheck.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 4 2 | Valid Record Count: 1 3 | Skipped Record Count: 0 4 | Large Record Count: 3 5 | -------------------------------------------------------------------------------- /SIT/features/05_guardrail/cdm.txt: -------------------------------------------------------------------------------- 1 | guardrailCheck com.datastax.cdm.job.GuardrailCheck migrate.properties 2 | -------------------------------------------------------------------------------- /SIT/features/05_guardrail/execute.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | #!/bin/bash -e 16 | 17 | workingDir="$1" 18 | cd "$workingDir" 19 | 20 | /local/cdm.sh -f cdm.txt -s guardrailCheck -d "$workingDir" > cdm.guardrailCheck.out 2>cdm.guardrailCheck.err 21 | /local/cdm-assert.sh -f cdm.guardrailCheck.out -a cdm.guardrailCheck.assert -d "$workingDir" 22 | -------------------------------------------------------------------------------- /SIT/features/05_guardrail/expected.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | SELECT count(*) FROM origin.feature_guardrail; 16 | -------------------------------------------------------------------------------- /SIT/features/05_guardrail/expected.out: -------------------------------------------------------------------------------- 1 | 2 | count 3 | ------- 4 | 4 5 | 6 | (1 rows) 7 | 8 | Warnings : 9 | Aggregation query used without partition key 10 | 11 | -------------------------------------------------------------------------------- /SIT/features/05_guardrail/migrate.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | spark.cdm.connect.origin.host cdm-sit-cass 16 | spark.cdm.connect.target.host cdm-sit-cass 17 | 18 | spark.cdm.schema.origin.keyspaceTable origin.feature_guardrail 19 | spark.cdm.perfops.numParts 1 20 | 21 | spark.cdm.feature.guardrail.colSizeInKB 1 22 | -------------------------------------------------------------------------------- /SIT/features/06_constant_column_remove/breakData.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | DELETE FROM target.feature_constant_column_remove WHERE key='key2'; 16 | UPDATE target.feature_constant_column_remove SET value='value999' WHERE key='key3'; 17 | 18 | # This upsert to origin will update the writetime on origin to be newer than target 19 | INSERT INTO origin.feature_constant_column_remove(const1, key, value, const2) VALUES (1, 'key1','valueA', 21); 20 | INSERT INTO origin.feature_constant_column_remove(const1, key, value, const2) VALUES (1, 'key2','valueB', 22); 21 | INSERT INTO origin.feature_constant_column_remove(const1, key, value, const2) VALUES (1, 'key3','valueC', 23); 22 | 23 | SELECT * FROM target.feature_constant_column_remove; 24 | -------------------------------------------------------------------------------- /SIT/features/06_constant_column_remove/cdm.fixData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 3 2 | Mismatch Record Count: 1 3 | Corrected Mismatch Record Count: 1 4 | Missing Record Count: 1 5 | Corrected Missing Record Count: 1 6 | Valid Record Count: 1 7 | Skipped Record Count: 0 8 | Error Record Count: 0 9 | -------------------------------------------------------------------------------- /SIT/features/06_constant_column_remove/cdm.migrateData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 3 2 | Skipped Record Count: 0 3 | Write Record Count: 3 4 | Error Record Count: 0 5 | -------------------------------------------------------------------------------- /SIT/features/06_constant_column_remove/cdm.txt: -------------------------------------------------------------------------------- 1 | migrateData com.datastax.cdm.job.Migrate migrate.properties 2 | validateData com.datastax.cdm.job.DiffData migrate.properties 3 | fixData com.datastax.cdm.job.DiffData fix.properties 4 | -------------------------------------------------------------------------------- /SIT/features/06_constant_column_remove/cdm.validateData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 3 2 | Mismatch Record Count: 0 3 | Corrected Mismatch Record Count: 0 4 | Missing Record Count: 0 5 | Corrected Missing Record Count: 0 6 | Valid Record Count: 3 7 | Skipped Record Count: 0 8 | Error Record Count: 0 9 | -------------------------------------------------------------------------------- /SIT/features/06_constant_column_remove/execute.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | #!/bin/bash -e 16 | 17 | workingDir="$1" 18 | cd "$workingDir" 19 | 20 | /local/cdm.sh -f cdm.txt -s migrateData -d "$workingDir" > cdm.migrateData.out 2>cdm.migrateData.err 21 | /local/cdm-assert.sh -f cdm.migrateData.out -a cdm.migrateData.assert -d "$workingDir" 22 | 23 | /local/cdm.sh -f cdm.txt -s validateData -d "$workingDir" > cdm.validateData.out 2>cdm.validateData.err 24 | /local/cdm-assert.sh -f cdm.validateData.out -a cdm.validateData.assert -d "$workingDir" 25 | 26 | cqlsh -u $CASS_USERNAME -p $CASS_PASSWORD $CASS_CLUSTER -f $workingDir/breakData.cql > $workingDir/breakData.out 2> $workingDir/breakData.err 27 | 28 | /local/cdm.sh -f cdm.txt -s fixData -d "$workingDir" > cdm.fixData.out 2>cdm.fixData.err 29 | /local/cdm-assert.sh -f cdm.fixData.out -a cdm.fixData.assert -d "$workingDir" -------------------------------------------------------------------------------- /SIT/features/06_constant_column_remove/expected.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | SELECT * FROM target.feature_constant_column_remove; 16 | -------------------------------------------------------------------------------- /SIT/features/06_constant_column_remove/expected.out: -------------------------------------------------------------------------------- 1 | 2 | key | value 3 | ------+-------- 4 | key1 | valueA 5 | key3 | valueC 6 | key2 | valueB 7 | 8 | (3 rows) 9 | -------------------------------------------------------------------------------- /SIT/features/06_constant_column_remove/fix.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | spark.cdm.connect.origin.host cdm-sit-cass 16 | spark.cdm.connect.target.host cdm-sit-cass 17 | 18 | spark.cdm.schema.origin.keyspaceTable origin.feature_constant_column_remove 19 | spark.cdm.schema.target.keyspaceTable target.feature_constant_column_remove 20 | spark.cdm.perfops.numParts 1 21 | 22 | spark.cdm.autocorrect.missing true 23 | spark.cdm.autocorrect.mismatch true 24 | -------------------------------------------------------------------------------- /SIT/features/06_constant_column_remove/migrate.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | spark.cdm.connect.origin.host cdm-sit-cass 16 | spark.cdm.connect.target.host cdm-sit-cass 17 | 18 | spark.cdm.schema.origin.keyspaceTable origin.feature_constant_column_remove 19 | spark.cdm.schema.target.keyspaceTable target.feature_constant_column_remove 20 | spark.cdm.perfops.numParts 1 21 | -------------------------------------------------------------------------------- /SIT/features/06_constant_column_remove/setup.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | DROP TABLE IF EXISTS origin.feature_constant_column_remove; 16 | CREATE TABLE origin.feature_constant_column_remove(const1 int, key text, value text, const2 int, PRIMARY KEY (const1, key)); 17 | INSERT INTO origin.feature_constant_column_remove(const1, key, value, const2) VALUES (1, 'key1','valueA', 21); 18 | INSERT INTO origin.feature_constant_column_remove(const1, key, value, const2) VALUES (1, 'key2','valueB', 22); 19 | INSERT INTO origin.feature_constant_column_remove(const1, key, value, const2) VALUES (1, 'key3','valueC', 23); 20 | 21 | DROP TABLE IF EXISTS target.feature_constant_column_remove; 22 | CREATE TABLE target.feature_constant_column_remove(key text, value text, PRIMARY KEY (key)); 23 | -------------------------------------------------------------------------------- /SIT/features/07_constant_column_replace/breakData.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | DELETE FROM target.feature_constant_column_replace WHERE const3=2 AND key='key2'; 16 | UPDATE target.feature_constant_column_replace SET value='value999' WHERE const3=2 AND key='key3'; 17 | 18 | # This upsert to origin will update the writetime on origin to be newer than target 19 | INSERT INTO origin.feature_constant_column_replace(const1, key, value, const2) VALUES (1, 'key1','valueA', 21); 20 | INSERT INTO origin.feature_constant_column_replace(const1, key, value, const2) VALUES (1, 'key2','valueB', 22); 21 | INSERT INTO origin.feature_constant_column_replace(const1, key, value, const2) VALUES (1, 'key3','valueC', 23); 22 | 23 | SELECT * FROM target.feature_constant_column_replace; 24 | 25 | -------------------------------------------------------------------------------- /SIT/features/07_constant_column_replace/cdm.fixData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 3 2 | Mismatch Record Count: 1 3 | Corrected Mismatch Record Count: 1 4 | Missing Record Count: 1 5 | Corrected Missing Record Count: 1 6 | Valid Record Count: 1 7 | Skipped Record Count: 0 8 | Error Record Count: 0 9 | -------------------------------------------------------------------------------- /SIT/features/07_constant_column_replace/cdm.migrateData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 3 2 | Skipped Record Count: 0 3 | Write Record Count: 3 4 | Error Record Count: 0 5 | -------------------------------------------------------------------------------- /SIT/features/07_constant_column_replace/cdm.txt: -------------------------------------------------------------------------------- 1 | migrateData com.datastax.cdm.job.Migrate migrate.properties 2 | validateData com.datastax.cdm.job.DiffData migrate.properties 3 | fixData com.datastax.cdm.job.DiffData fix.properties 4 | -------------------------------------------------------------------------------- /SIT/features/07_constant_column_replace/cdm.validateData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 3 2 | Mismatch Record Count: 0 3 | Corrected Mismatch Record Count: 0 4 | Missing Record Count: 0 5 | Corrected Missing Record Count: 0 6 | Valid Record Count: 3 7 | Skipped Record Count: 0 8 | Error Record Count: 0 9 | -------------------------------------------------------------------------------- /SIT/features/07_constant_column_replace/execute.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | #!/bin/bash -e 16 | 17 | workingDir="$1" 18 | cd "$workingDir" 19 | 20 | /local/cdm.sh -f cdm.txt -s migrateData -d "$workingDir" > cdm.migrateData.out 2>cdm.migrateData.err 21 | /local/cdm-assert.sh -f cdm.migrateData.out -a cdm.migrateData.assert -d "$workingDir" 22 | 23 | /local/cdm.sh -f cdm.txt -s validateData -d "$workingDir" > cdm.validateData.out 2>cdm.validateData.err 24 | /local/cdm-assert.sh -f cdm.validateData.out -a cdm.validateData.assert -d "$workingDir" 25 | 26 | cqlsh -u $CASS_USERNAME -p $CASS_PASSWORD $CASS_CLUSTER -f $workingDir/breakData.cql > $workingDir/breakData.out 2> $workingDir/breakData.err 27 | 28 | /local/cdm.sh -f cdm.txt -s fixData -d "$workingDir" > cdm.fixData.out 2>cdm.fixData.err 29 | /local/cdm-assert.sh -f cdm.fixData.out -a cdm.fixData.assert -d "$workingDir" -------------------------------------------------------------------------------- /SIT/features/07_constant_column_replace/expected.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | SELECT * FROM target.feature_constant_column_replace; 16 | -------------------------------------------------------------------------------- /SIT/features/07_constant_column_replace/expected.out: -------------------------------------------------------------------------------- 1 | 2 | const3 | key | const4 | value 3 | --------+------+--------+-------- 4 | 2 | key1 | 31 | valueA 5 | 2 | key2 | 31 | valueB 6 | 2 | key3 | 31 | valueC 7 | 8 | (3 rows) 9 | -------------------------------------------------------------------------------- /SIT/features/07_constant_column_replace/fix.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | spark.cdm.connect.origin.host cdm-sit-cass 16 | spark.cdm.connect.target.host cdm-sit-cass 17 | 18 | spark.cdm.schema.origin.keyspaceTable origin.feature_constant_column_replace 19 | spark.cdm.schema.target.keyspaceTable target.feature_constant_column_replace 20 | spark.cdm.perfops.numParts 1 21 | 22 | spark.cdm.feature.constantColumns.names const3,const4 23 | spark.cdm.feature.constantColumns.values 2,31 24 | 25 | spark.cdm.autocorrect.missing true 26 | spark.cdm.autocorrect.mismatch true 27 | -------------------------------------------------------------------------------- /SIT/features/07_constant_column_replace/migrate.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | spark.cdm.connect.origin.host cdm-sit-cass 16 | spark.cdm.connect.target.host cdm-sit-cass 17 | 18 | spark.cdm.schema.origin.keyspaceTable origin.feature_constant_column_replace 19 | spark.cdm.schema.target.keyspaceTable target.feature_constant_column_replace 20 | spark.cdm.perfops.numParts 1 21 | 22 | spark.cdm.feature.constantColumns.names const3,const4 23 | spark.cdm.feature.constantColumns.values 2,31 24 | 25 | -------------------------------------------------------------------------------- /SIT/features/07_constant_column_replace/setup.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | DROP TABLE IF EXISTS origin.feature_constant_column_replace; 16 | CREATE TABLE origin.feature_constant_column_replace(const1 int, key text, value text, const2 int, PRIMARY KEY (const1, key)); 17 | INSERT INTO origin.feature_constant_column_replace(const1, key, value, const2) VALUES (1, 'key1','valueA', 21); 18 | INSERT INTO origin.feature_constant_column_replace(const1, key, value, const2) VALUES (1, 'key2','valueB', 22); 19 | INSERT INTO origin.feature_constant_column_replace(const1, key, value, const2) VALUES (1, 'key3','valueC', 23); 20 | 21 | DROP TABLE IF EXISTS target.feature_constant_column_replace; 22 | CREATE TABLE target.feature_constant_column_replace(const3 int, key text, value text, const4 int, PRIMARY KEY (const3, key)); 23 | -------------------------------------------------------------------------------- /SIT/regression/01_explode_map_with_constants/breakData.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | DELETE FROM target.feature_explode_map_with_constants WHERE const1='abcd' AND const2=1234 AND key='key2'; 16 | UPDATE target.feature_explode_map_with_constants SET time=7398730800000 WHERE const1='abcd' AND const2=1234 AND key='key3' AND fruit='apples'; 17 | UPDATE target.feature_explode_map_with_constants SET fruit_qty=999 WHERE const1='abcd' AND const2=1234 AND key='key3' AND fruit='oranges'; 18 | DELETE FROM target.feature_explode_map_with_constants WHERE const1='abcd' AND const2=1234 AND key='key3' AND fruit='kiwi'; 19 | 20 | # This upsert to origin will update the writetime on origin to be newer than target 21 | INSERT INTO origin.feature_explode_map_with_constants(key,time,fruits) VALUES ('key1','1087383600000', {'apples': 3, 'oranges': 5, 'bananas': 2, 'grapes': 11}); 22 | INSERT INTO origin.feature_explode_map_with_constants(key,time,fruits) VALUES ('key2','1087383600000', {'apples': 4, 'oranges': 6, 'bananas': 3, 'pears': 7}); 23 | INSERT INTO origin.feature_explode_map_with_constants(key,time,fruits) VALUES ('key3','1087383600000', {'apples': 5, 'oranges': 7, 'bananas': 4, 'kiwi': 42}); 24 | 25 | SELECT * FROM target.feature_explode_map_with_constants; 26 | -------------------------------------------------------------------------------- /SIT/regression/01_explode_map_with_constants/cdm.fixData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 3 2 | Mismatch Record Count: 2 3 | Corrected Mismatch Record Count: 2 4 | Missing Record Count: 5 5 | Corrected Missing Record Count: 5 6 | Valid Record Count: 5 7 | Skipped Record Count: 0 8 | Error Record Count: 0 9 | -------------------------------------------------------------------------------- /SIT/regression/01_explode_map_with_constants/cdm.migrateData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 3 2 | Skipped Record Count: 0 3 | Write Record Count: 12 4 | Error Record Count: 0 5 | -------------------------------------------------------------------------------- /SIT/regression/01_explode_map_with_constants/cdm.txt: -------------------------------------------------------------------------------- 1 | migrateData com.datastax.cdm.job.Migrate migrate.properties 2 | validateData com.datastax.cdm.job.DiffData migrate.properties 3 | fixData com.datastax.cdm.job.DiffData fix.properties 4 | -------------------------------------------------------------------------------- /SIT/regression/01_explode_map_with_constants/cdm.validateData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 3 2 | Mismatch Record Count: 0 3 | Corrected Mismatch Record Count: 0 4 | Missing Record Count: 0 5 | Corrected Missing Record Count: 0 6 | Valid Record Count: 12 7 | Skipped Record Count: 0 8 | Error Record Count: 0 9 | -------------------------------------------------------------------------------- /SIT/regression/01_explode_map_with_constants/execute.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | #!/bin/bash -e 16 | 17 | workingDir="$1" 18 | cd "$workingDir" 19 | 20 | /local/cdm.sh -f cdm.txt -s migrateData -d "$workingDir" > cdm.migrateData.out 2>cdm.migrateData.err 21 | /local/cdm-assert.sh -f cdm.migrateData.out -a cdm.migrateData.assert -d "$workingDir" 22 | 23 | /local/cdm.sh -f cdm.txt -s validateData -d "$workingDir" > cdm.validateData.out 2>cdm.validateData.err 24 | /local/cdm-assert.sh -f cdm.validateData.out -a cdm.validateData.assert -d "$workingDir" 25 | 26 | cqlsh -u $CASS_USERNAME -p $CASS_PASSWORD $CASS_CLUSTER -f $workingDir/breakData.cql > $workingDir/breakData.out 2> $workingDir/breakData.err 27 | 28 | /local/cdm.sh -f cdm.txt -s fixData -d "$workingDir" > cdm.fixData.out 2>cdm.fixData.err 29 | /local/cdm-assert.sh -f cdm.fixData.out -a cdm.fixData.assert -d "$workingDir" 30 | -------------------------------------------------------------------------------- /SIT/regression/01_explode_map_with_constants/expected.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | SELECT * FROM target.feature_explode_map_with_constants; 16 | -------------------------------------------------------------------------------- /SIT/regression/01_explode_map_with_constants/expected.out: -------------------------------------------------------------------------------- 1 | 2 | const1 | const2 | key | fruit | fruit_qty | time 3 | --------+--------+------+---------+-----------+--------------------------------- 4 | abcd | 1234 | key1 | apples | 3 | 2004-06-16 11:00:00.000000+0000 5 | abcd | 1234 | key1 | bananas | 2 | 2004-06-16 11:00:00.000000+0000 6 | abcd | 1234 | key1 | grapes | 11 | 2004-06-16 11:00:00.000000+0000 7 | abcd | 1234 | key1 | oranges | 5 | 2004-06-16 11:00:00.000000+0000 8 | abcd | 1234 | key3 | apples | 5 | 2004-06-16 11:00:00.000000+0000 9 | abcd | 1234 | key3 | bananas | 4 | 2004-06-16 11:00:00.000000+0000 10 | abcd | 1234 | key3 | kiwi | 42 | 2004-06-16 11:00:00.000000+0000 11 | abcd | 1234 | key3 | oranges | 7 | 2004-06-16 11:00:00.000000+0000 12 | abcd | 1234 | key2 | apples | 4 | 2004-06-16 11:00:00.000000+0000 13 | abcd | 1234 | key2 | bananas | 3 | 2004-06-16 11:00:00.000000+0000 14 | abcd | 1234 | key2 | oranges | 6 | 2004-06-16 11:00:00.000000+0000 15 | abcd | 1234 | key2 | pears | 7 | 2004-06-16 11:00:00.000000+0000 16 | 17 | (12 rows) 18 | -------------------------------------------------------------------------------- /SIT/regression/01_explode_map_with_constants/fix.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | spark.cdm.connect.origin.host cdm-sit-cass 16 | spark.cdm.connect.target.host cdm-sit-cass 17 | 18 | spark.cdm.schema.origin.keyspaceTable origin.feature_explode_map_with_constants 19 | spark.cdm.schema.target.keyspaceTable target.feature_explode_map_with_constants 20 | spark.cdm.perfops.numParts 1 21 | 22 | spark.cdm.feature.explodeMap.origin.name fruits 23 | spark.cdm.feature.explodeMap.target.name.key fruit 24 | spark.cdm.feature.explodeMap.target.name.value fruit_qty 25 | 26 | spark.cdm.feature.constantColumns.names const1,const2 27 | spark.cdm.feature.constantColumns.values 'abcd',1234 28 | 29 | spark.cdm.transform.codecs TIMESTAMP_STRING_MILLIS 30 | 31 | spark.cdm.autocorrect.missing true 32 | spark.cdm.autocorrect.mismatch true 33 | 34 | -------------------------------------------------------------------------------- /SIT/regression/01_explode_map_with_constants/migrate.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | spark.cdm.connect.origin.host cdm-sit-cass 16 | spark.cdm.connect.target.host cdm-sit-cass 17 | 18 | spark.cdm.schema.origin.keyspaceTable origin.feature_explode_map_with_constants 19 | spark.cdm.schema.target.keyspaceTable target.feature_explode_map_with_constants 20 | spark.cdm.perfops.numParts 1 21 | 22 | spark.cdm.feature.explodeMap.origin.name fruits 23 | spark.cdm.feature.explodeMap.target.name.key fruit 24 | spark.cdm.feature.explodeMap.target.name.value fruit_qty 25 | 26 | spark.cdm.feature.constantColumns.names const1,const2 27 | spark.cdm.feature.constantColumns.values 'abcd',1234 28 | 29 | spark.cdm.transform.codecs TIMESTAMP_STRING_MILLIS 30 | 31 | 32 | -------------------------------------------------------------------------------- /SIT/regression/01_explode_map_with_constants/setup.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | DROP TABLE IF EXISTS origin.feature_explode_map_with_constants; 16 | CREATE TABLE origin.feature_explode_map_with_constants(key text, time text, fruits map, PRIMARY KEY (key)); 17 | INSERT INTO origin.feature_explode_map_with_constants(key,time,fruits) VALUES ('key1','1087383600000', {'apples': 3, 'oranges': 5, 'bananas': 2, 'grapes': 11}); 18 | INSERT INTO origin.feature_explode_map_with_constants(key,time,fruits) VALUES ('key2','1087383600000', {'apples': 4, 'oranges': 6, 'bananas': 3, 'pears': 7}); 19 | INSERT INTO origin.feature_explode_map_with_constants(key,time,fruits) VALUES ('key3','1087383600000', {'apples': 5, 'oranges': 7, 'bananas': 4, 'kiwi': 42}); 20 | 21 | DROP TABLE IF EXISTS target.feature_explode_map_with_constants; 22 | CREATE TABLE target.feature_explode_map_with_constants(const1 text, const2 int, key text, fruit text, time timestamp, fruit_qty int, PRIMARY KEY ((const1,const2,key), fruit)); 23 | -------------------------------------------------------------------------------- /SIT/regression/02_ColumnRenameWithConstantsAndExplode/breakData.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | DELETE FROM target.column_rename_with_constants_and_explode WHERE customer='CUSTOMER' AND year=2023 AND key_name='key2'; 16 | UPDATE target.column_rename_with_constants_and_explode SET fruit_qty=999 WHERE customer='CUSTOMER' AND year=2023 AND key_name='key3' AND fruit='oranges'; 17 | DELETE FROM target.column_rename_with_constants_and_explode WHERE customer='CUSTOMER' AND year=2023 AND key_name='key3' AND fruit='kiwi'; 18 | SELECT * FROM target.column_rename_with_constants_and_explode; 19 | 20 | -------------------------------------------------------------------------------- /SIT/regression/02_ColumnRenameWithConstantsAndExplode/cdm.fixData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 3 2 | Mismatch Record Count: 1 3 | Corrected Mismatch Record Count: 1 4 | Missing Record Count: 5 5 | Corrected Missing Record Count: 5 6 | Valid Record Count: 6 7 | Skipped Record Count: 0 8 | Error Record Count: 0 9 | -------------------------------------------------------------------------------- /SIT/regression/02_ColumnRenameWithConstantsAndExplode/cdm.migrateData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 3 2 | Skipped Record Count: 0 3 | Write Record Count: 12 4 | Error Record Count: 0 5 | -------------------------------------------------------------------------------- /SIT/regression/02_ColumnRenameWithConstantsAndExplode/cdm.txt: -------------------------------------------------------------------------------- 1 | migrateData com.datastax.cdm.job.Migrate migrate.properties 2 | validateData com.datastax.cdm.job.DiffData migrate.properties 3 | fixData com.datastax.cdm.job.DiffData fix.properties 4 | -------------------------------------------------------------------------------- /SIT/regression/02_ColumnRenameWithConstantsAndExplode/cdm.validateData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 3 2 | Mismatch Record Count: 0 3 | Corrected Mismatch Record Count: 0 4 | Missing Record Count: 0 5 | Corrected Missing Record Count: 0 6 | Valid Record Count: 12 7 | Skipped Record Count: 0 8 | Error Record Count: 0 9 | -------------------------------------------------------------------------------- /SIT/regression/02_ColumnRenameWithConstantsAndExplode/execute.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | #!/bin/bash -e 16 | 17 | workingDir="$1" 18 | cd "$workingDir" 19 | 20 | /local/cdm.sh -f cdm.txt -s migrateData -d "$workingDir" > cdm.migrateData.out 2>cdm.migrateData.err 21 | /local/cdm-assert.sh -f cdm.migrateData.out -a cdm.migrateData.assert -d "$workingDir" 22 | 23 | /local/cdm.sh -f cdm.txt -s validateData -d "$workingDir" > cdm.validateData.out 2>cdm.validateData.err 24 | /local/cdm-assert.sh -f cdm.validateData.out -a cdm.validateData.assert -d "$workingDir" 25 | 26 | cqlsh -u $CASS_USERNAME -p $CASS_PASSWORD $CASS_CLUSTER -f $workingDir/breakData.cql > $workingDir/breakData.out 2> $workingDir/breakData.err 27 | 28 | /local/cdm.sh -f cdm.txt -s fixData -d "$workingDir" > cdm.fixData.out 2>cdm.fixData.err 29 | /local/cdm-assert.sh -f cdm.fixData.out -a cdm.fixData.assert -d "$workingDir" 30 | -------------------------------------------------------------------------------- /SIT/regression/02_ColumnRenameWithConstantsAndExplode/expected.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | SELECT * FROM target.column_rename_with_constants_and_explode; 16 | -------------------------------------------------------------------------------- /SIT/regression/02_ColumnRenameWithConstantsAndExplode/expected.out: -------------------------------------------------------------------------------- 1 | 2 | customer | year | key_name | fruit | fruit_qty 3 | ----------+------+----------+---------+----------- 4 | CUSTOMER | 2023 | key2 | apples | 4 5 | CUSTOMER | 2023 | key2 | bananas | 3 6 | CUSTOMER | 2023 | key2 | oranges | 6 7 | CUSTOMER | 2023 | key2 | pears | 7 8 | CUSTOMER | 2023 | key1 | apples | 3 9 | CUSTOMER | 2023 | key1 | bananas | 2 10 | CUSTOMER | 2023 | key1 | grapes | 11 11 | CUSTOMER | 2023 | key1 | oranges | 5 12 | CUSTOMER | 2023 | key3 | apples | 5 13 | CUSTOMER | 2023 | key3 | bananas | 4 14 | CUSTOMER | 2023 | key3 | kiwi | 42 15 | CUSTOMER | 2023 | key3 | oranges | 7 16 | 17 | (12 rows) 18 | -------------------------------------------------------------------------------- /SIT/regression/02_ColumnRenameWithConstantsAndExplode/fix.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | spark.cdm.connect.origin.host cdm-sit-cass 16 | spark.cdm.connect.target.host cdm-sit-cass 17 | 18 | spark.cdm.schema.origin.keyspaceTable origin."ColumnRenameWithConstantsAndExplode" 19 | spark.cdm.schema.target.keyspaceTable target.column_rename_with_constants_and_explode 20 | spark.cdm.perfops.numParts 1 21 | 22 | spark.cdm.schema.origin.column.names.to.target "key-name":key_name 23 | spark.cdm.feature.explodeMap.origin.name "fruit-map" 24 | spark.cdm.feature.explodeMap.target.name.key fruit 25 | spark.cdm.feature.explodeMap.target.name.value fruit_qty 26 | 27 | spark.cdm.feature.constantColumns.names customer,year 28 | spark.cdm.feature.constantColumns.values 'CUSTOMER',2023 29 | 30 | spark.cdm.autocorrect.missing true 31 | spark.cdm.autocorrect.mismatch true 32 | 33 | -------------------------------------------------------------------------------- /SIT/regression/02_ColumnRenameWithConstantsAndExplode/migrate.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | spark.cdm.connect.origin.host cdm-sit-cass 16 | spark.cdm.connect.target.host cdm-sit-cass 17 | 18 | spark.cdm.schema.origin.keyspaceTable origin."ColumnRenameWithConstantsAndExplode" 19 | spark.cdm.schema.target.keyspaceTable target.column_rename_with_constants_and_explode 20 | spark.cdm.perfops.numParts 1 21 | 22 | spark.cdm.schema.origin.column.names.to.target "key-name":key_name 23 | 24 | spark.cdm.feature.explodeMap.origin.name "fruit-map" 25 | spark.cdm.feature.explodeMap.target.name.key fruit 26 | spark.cdm.feature.explodeMap.target.name.value fruit_qty 27 | 28 | spark.cdm.feature.constantColumns.names customer,year 29 | spark.cdm.feature.constantColumns.values 'CUSTOMER',2023 30 | -------------------------------------------------------------------------------- /SIT/regression/02_ColumnRenameWithConstantsAndExplode/setup.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | DROP TABLE IF EXISTS origin."ColumnRenameWithConstantsAndExplode"; 16 | CREATE TABLE origin."ColumnRenameWithConstantsAndExplode" ( 17 | "key-name" text PRIMARY KEY, 18 | "fruit-map" map 19 | ); 20 | 21 | INSERT INTO origin."ColumnRenameWithConstantsAndExplode" ("key-name","fruit-map") VALUES ('key1', {'apples': 3, 'oranges': 5, 'bananas': 2, 'grapes': 11}); 22 | INSERT INTO origin."ColumnRenameWithConstantsAndExplode" ("key-name","fruit-map") VALUES ('key2', {'apples': 4, 'oranges': 6, 'bananas': 3, 'pears': 7}); 23 | INSERT INTO origin."ColumnRenameWithConstantsAndExplode" ("key-name","fruit-map") VALUES ('key3', {'apples': 5, 'oranges': 7, 'bananas': 4, 'kiwi': 42}); 24 | 25 | DROP TABLE IF EXISTS target.column_rename_with_constants_and_explode; 26 | CREATE TABLE target.column_rename_with_constants_and_explode ( 27 | customer text, 28 | year int, 29 | key_name text, 30 | fruit text, 31 | fruit_qty int, 32 | PRIMARY KEY ((customer, year, key_name), fruit) 33 | ); 34 | -------------------------------------------------------------------------------- /SIT/regression/03_performance/breakData.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | DELETE FROM target.regression_performance WHERE pk_id = 600; 16 | SELECT * FROM target.regression_performance WHERE pk_id >= 400 AND pk_id < 700 ALLOW FILTERING; 17 | -------------------------------------------------------------------------------- /SIT/regression/03_performance/cdm.fixData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 4000 2 | Mismatch Record Count: 100 3 | Corrected Mismatch Record Count: 100 4 | Missing Record Count: 100 5 | Corrected Missing Record Count: 100 6 | Valid Record Count: 3800 7 | Skipped Record Count: 0 8 | Error Record Count: 0 9 | -------------------------------------------------------------------------------- /SIT/regression/03_performance/cdm.migrateData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 4000 2 | Skipped Record Count: 0 3 | Write Record Count: 4000 4 | Error Record Count: 0 5 | -------------------------------------------------------------------------------- /SIT/regression/03_performance/cdm.txt: -------------------------------------------------------------------------------- 1 | migrateData com.datastax.cdm.job.Migrate migrate.properties 2 | validateData com.datastax.cdm.job.DiffData migrate.properties 3 | fixData com.datastax.cdm.job.DiffData fix.properties 4 | -------------------------------------------------------------------------------- /SIT/regression/03_performance/cdm.validateData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 4000 2 | Mismatch Record Count: 0 3 | Corrected Mismatch Record Count: 0 4 | Missing Record Count: 0 5 | Corrected Missing Record Count: 0 6 | Valid Record Count: 4000 7 | Skipped Record Count: 0 8 | Error Record Count: 0 9 | -------------------------------------------------------------------------------- /SIT/regression/03_performance/execute.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | #!/bin/bash -e 16 | 17 | workingDir="$1" 18 | cd "$workingDir" 19 | 20 | dsbulk load -url $workingDir/data.csv -h $CASS_CLUSTER -u $CASS_USERNAME -p $CASS_PASSWORD -k origin -t regression_performance 21 | 22 | /local/cdm.sh -f cdm.txt -s migrateData -d "$workingDir" > cdm.migrateData.out 2>cdm.migrateData.err 23 | /local/cdm-assert.sh -f cdm.migrateData.out -a cdm.migrateData.assert -d "$workingDir" 24 | 25 | /local/cdm.sh -f cdm.txt -s validateData -d "$workingDir" > cdm.validateData.out 2>cdm.validateData.err 26 | /local/cdm-assert.sh -f cdm.validateData.out -a cdm.validateData.assert -d "$workingDir" 27 | 28 | dsbulk load -url $workingDir/data_break.csv -h $CASS_CLUSTER -u $CASS_USERNAME -p $CASS_PASSWORD -k target -t regression_performance 29 | cqlsh -u $CASS_USERNAME -p $CASS_PASSWORD $CASS_CLUSTER -f $workingDir/breakData.cql > $workingDir/breakData.out 2> $workingDir/breakData.err 30 | 31 | # This upsert to origin will update the writetime on origin to be newer than target 32 | dsbulk load -url $workingDir/data.csv -h $CASS_CLUSTER -u $CASS_USERNAME -p $CASS_PASSWORD -k origin -t regression_performance 33 | 34 | /local/cdm.sh -f cdm.txt -s fixData -d "$workingDir" > cdm.fixData.out 2>cdm.fixData.err 35 | /local/cdm-assert.sh -f cdm.fixData.out -a cdm.fixData.assert -d "$workingDir" 36 | -------------------------------------------------------------------------------- /SIT/regression/03_performance/expected.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | SELECT * FROM target.regression_performance WHERE pk_id >= 400 AND pk_id < 700 ALLOW FILTERING; 16 | -------------------------------------------------------------------------------- /SIT/regression/03_performance/fix.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | spark.cdm.connect.origin.host cdm-sit-cass 16 | spark.cdm.connect.target.host cdm-sit-cass 17 | 18 | spark.cdm.schema.origin.keyspaceTable origin.regression_performance 19 | spark.cdm.schema.target.keyspaceTable target.regression_performance 20 | 21 | spark.cdm.perfops.numParts 32 22 | spark.cdm.perfops.batchSize 1 23 | 24 | spark.cdm.autocorrect.missing true 25 | spark.cdm.autocorrect.mismatch true 26 | -------------------------------------------------------------------------------- /SIT/regression/03_performance/migrate.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | spark.cdm.connect.origin.host cdm-sit-cass 16 | spark.cdm.connect.target.host cdm-sit-cass 17 | 18 | spark.cdm.schema.origin.keyspaceTable origin.regression_performance 19 | spark.cdm.schema.target.keyspaceTable target.regression_performance 20 | 21 | spark.cdm.perfops.numParts 32 22 | spark.cdm.perfops.batchSize 1 23 | -------------------------------------------------------------------------------- /SIT/regression/03_performance/setup.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | DROP TABLE IF EXISTS origin.regression_performance; 16 | CREATE TABLE origin.regression_performance (pk_id int, ck_id text, f1 text, PRIMARY KEY (pk_id, ck_id) ) WITH CLUSTERING ORDER BY (ck_id ASC); 17 | 18 | DROP TABLE IF EXISTS target.regression_performance; 19 | CREATE TABLE target.regression_performance (pk_id int, ck_id text, f1 text, PRIMARY KEY (pk_id, ck_id) ) WITH CLUSTERING ORDER BY (ck_id ASC); 20 | -------------------------------------------------------------------------------- /SIT/regression/04_null_ts_in_pk/cdm.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | #!/bin/bash -e 16 | 17 | cat <&2 17 | -------------------------------------------------------------------------------- /SIT/smoke/00_test_harness/expected.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); 3 | you may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | Unless required by applicable law or agreed to in writing, software 7 | distributed under the License is distributed on an "AS IS" BASIS, 8 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | See the License for the specific language governing permissions and 10 | limitations under the License. 11 | */ 12 | 13 | SELECT * FROM origin.smoke_test_harness; 14 | SELECT * FROM target.smoke_test_harness; 15 | -------------------------------------------------------------------------------- /SIT/smoke/00_test_harness/expected.out: -------------------------------------------------------------------------------- 1 | 2 | key | value 3 | ------+-------- 4 | key1 | valueA 5 | key2 | valueB 6 | 7 | (2 rows) 8 | 9 | key | value 10 | ------+-------- 11 | key1 | valueA 12 | key2 | valueB 13 | 14 | (2 rows) 15 | -------------------------------------------------------------------------------- /SIT/smoke/00_test_harness/setup.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); 3 | you may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | Unless required by applicable law or agreed to in writing, software 7 | distributed under the License is distributed on an "AS IS" BASIS, 8 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | See the License for the specific language governing permissions and 10 | limitations under the License. 11 | */ 12 | 13 | DROP TABLE IF EXISTS origin.smoke_test_harness; 14 | CREATE TABLE origin.smoke_test_harness(key text, value text, PRIMARY KEY (key)); 15 | INSERT INTO origin.smoke_test_harness(key,value) VALUES ('key1','valueA'); 16 | INSERT INTO origin.smoke_test_harness(key,value) VALUES ('key2','valueB'); 17 | 18 | DROP TABLE IF EXISTS target.smoke_test_harness; 19 | CREATE TABLE target.smoke_test_harness(key text, value text, PRIMARY KEY (key)); 20 | INSERT INTO target.smoke_test_harness(key,value) VALUES ('key1','valueA'); 21 | INSERT INTO target.smoke_test_harness(key,value) VALUES ('key2','valueB'); 22 | -------------------------------------------------------------------------------- /SIT/smoke/01_basic_kvp/cdm.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | #!/bin/bash -e 16 | 17 | cat < $workingDir/other.breakData.out 2> $workingDir/other.breakData.err 23 | 24 | /local/cdm.sh -f cdm.txt -s validateData -d "$workingDir" 25 | 26 | -------------------------------------------------------------------------------- /SIT/smoke/02_autocorrect_kvp/expected.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | Unless required by applicable law or agreed to in writing, software 7 | distributed under the License is distributed on an "AS IS" BASIS, 8 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | See the License for the specific language governing permissions and 10 | limitations under the License. 11 | */ 12 | 13 | SELECT * FROM target.smoke_autocorrect_kvp; 14 | -------------------------------------------------------------------------------- /SIT/smoke/02_autocorrect_kvp/expected.out: -------------------------------------------------------------------------------- 1 | 2 | key | value 3 | ------+-------- 4 | key1 | valueA 5 | key3 | valueC 6 | key2 | valueB 7 | 8 | (3 rows) 9 | -------------------------------------------------------------------------------- /SIT/smoke/02_autocorrect_kvp/migrate.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | spark.cdm.connect.origin.host cdm-sit-cass 16 | spark.cdm.connect.target.host cdm-sit-cass 17 | 18 | spark.cdm.schema.origin.keyspaceTable origin.smoke_autocorrect_kvp 19 | spark.cdm.schema.target.keyspaceTable target.smoke_autocorrect_kvp 20 | spark.cdm.perfops.numParts 1 21 | 22 | spark.cdm.autocorrect.missing true 23 | spark.cdm.autocorrect.mismatch true 24 | 25 | -------------------------------------------------------------------------------- /SIT/smoke/02_autocorrect_kvp/setup.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | Unless required by applicable law or agreed to in writing, software 7 | distributed under the License is distributed on an "AS IS" BASIS, 8 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | See the License for the specific language governing permissions and 10 | limitations under the License. 11 | */ 12 | 13 | DROP TABLE IF EXISTS origin.smoke_autocorrect_kvp; 14 | CREATE TABLE origin.smoke_autocorrect_kvp(key text, value text, PRIMARY KEY (key)); 15 | INSERT INTO origin.smoke_autocorrect_kvp(key,value) VALUES ('key1','valueA'); 16 | INSERT INTO origin.smoke_autocorrect_kvp(key,value) VALUES ('key2','valueB'); 17 | INSERT INTO origin.smoke_autocorrect_kvp(key,value) VALUES ('key3','valueC'); 18 | 19 | DROP TABLE IF EXISTS target.smoke_autocorrect_kvp; 20 | CREATE TABLE target.smoke_autocorrect_kvp(key text, value text, PRIMARY KEY (key)); 21 | -------------------------------------------------------------------------------- /SIT/smoke/03_ttl_writetime/alternateCheckResults.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | #!/bin/bash 16 | # We need TTL to go down by at least 1 second, e.g. make 60000 to 59999 17 | sleep 1 18 | docker exec ${DOCKER_CASS} /bin/bash -c "cqlsh -u $CASS_USERNAME -p $CASS_PASSWORD -f $testDir/expected.cql | tr -d ' ' | cut -c1-46" 19 | -------------------------------------------------------------------------------- /SIT/smoke/03_ttl_writetime/cdm.fixData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 8 2 | Mismatch Record Count: 4 3 | Corrected Mismatch Record Count: 4 4 | Missing Record Count: 1 5 | Corrected Missing Record Count: 1 6 | Valid Record Count: 3 7 | Skipped Record Count: 0 8 | Error Record Count: 0 9 | -------------------------------------------------------------------------------- /SIT/smoke/03_ttl_writetime/cdm.migrateData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 8 2 | Skipped Record Count: 0 3 | Write Record Count: 8 4 | Error Record Count: 0 5 | -------------------------------------------------------------------------------- /SIT/smoke/03_ttl_writetime/cdm.txt: -------------------------------------------------------------------------------- 1 | migrateData com.datastax.cdm.job.Migrate migrate.properties 2 | validateData com.datastax.cdm.job.DiffData migrate.properties 3 | fixData com.datastax.cdm.job.DiffData fix.properties 4 | -------------------------------------------------------------------------------- /SIT/smoke/03_ttl_writetime/cdm.validateData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 8 2 | Mismatch Record Count: 0 3 | Corrected Mismatch Record Count: 0 4 | Missing Record Count: 0 5 | Corrected Missing Record Count: 0 6 | Valid Record Count: 8 7 | Skipped Record Count: 0 8 | Error Record Count: 0 9 | -------------------------------------------------------------------------------- /SIT/smoke/03_ttl_writetime/execute.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | #!/bin/bash -e 16 | 17 | workingDir="$1" 18 | cd "$workingDir" 19 | 20 | /local/cdm.sh -f cdm.txt -s migrateData -d "$workingDir" > cdm.migrateData.out 2>cdm.migrateData.err 21 | /local/cdm-assert.sh -f cdm.migrateData.out -a cdm.migrateData.assert -d "$workingDir" 22 | 23 | /local/cdm.sh -f cdm.txt -s validateData -d "$workingDir" > cdm.validateData.out 2>cdm.validateData.err 24 | /local/cdm-assert.sh -f cdm.validateData.out -a cdm.validateData.assert -d "$workingDir" 25 | 26 | cqlsh -u $CASS_USERNAME -p $CASS_PASSWORD $CASS_CLUSTER -f $workingDir/breakData.cql > $workingDir/breakData.out 2> $workingDir/breakData.err 27 | 28 | /local/cdm.sh -f cdm.txt -s fixData -d "$workingDir" > cdm.fixData.out 2>cdm.fixData.err 29 | /local/cdm-assert.sh -f cdm.fixData.out -a cdm.fixData.assert -d "$workingDir" 30 | -------------------------------------------------------------------------------- /SIT/smoke/03_ttl_writetime/expected.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | Unless required by applicable law or agreed to in writing, software 7 | distributed under the License is distributed on an "AS IS" BASIS, 8 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | See the License for the specific language governing permissions and 10 | limitations under the License. 11 | */ 12 | 13 | SELECT key,t_col1,tw_col2,w_col3,col4, writetime(col4) as wt_col4, ttl(col4) as ttl_col4 FROM target.smoke_ttl_writetime; 14 | -------------------------------------------------------------------------------- /SIT/smoke/03_ttl_writetime/expected.out: -------------------------------------------------------------------------------- 1 | 2 | key|t_col1|tw_col2|w_col3|col4|wt_col4|ttl_col 3 | ---------+--------+---------+--------+------+- 4 | record3|AAAA|BBBB|CCCC|DDDD|1087383720000000|5 5 | record5|A---|B---|CCCC|DDDD|1087384200000000|6 6 | record2|AAAA|BBBB|CCCC|DDDD|1087383720000000|5 7 | record6|AAAA|BBBB|C---|DDDD|1087384200000000|5 8 | record1|AAAA|BBBB|CCCC|DDDD|1087383720000000|5 9 | record8|A---|BBBB|CCCC|DDDD|1087383720000000|6 10 | record7|AAAA|B---|CCCC|DDDD|1087384200000000|6 11 | record4|AAAA|BBBB|CCCC|DDDD|1087384200000000|3 12 | 13 | (8rows) 14 | -------------------------------------------------------------------------------- /SIT/smoke/03_ttl_writetime/fix.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | spark.cdm.connect.origin.host cdm-sit-cass 16 | spark.cdm.connect.target.host cdm-sit-cass 17 | 18 | spark.cdm.schema.origin.keyspaceTable origin.smoke_ttl_writetime 19 | spark.cdm.schema.target.keyspaceTable target.smoke_ttl_writetime 20 | spark.cdm.perfops.numParts 1 21 | 22 | spark.cdm.schema.origin.column.ttl.names t_col1,tw_col2 23 | spark.cdm.schema.origin.column.writetime.names tw_col2,w_col3 24 | 25 | spark.cdm.autocorrect.missing true 26 | spark.cdm.autocorrect.mismatch true 27 | 28 | -------------------------------------------------------------------------------- /SIT/smoke/03_ttl_writetime/migrate.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | spark.cdm.connect.origin.host cdm-sit-cass 16 | spark.cdm.connect.target.host cdm-sit-cass 17 | 18 | spark.cdm.schema.origin.keyspaceTable origin.smoke_ttl_writetime 19 | spark.cdm.schema.target.keyspaceTable target.smoke_ttl_writetime 20 | spark.cdm.perfops.numParts 1 21 | 22 | spark.cdm.schema.origin.column.ttl.names t_col1,tw_col2 23 | spark.cdm.schema.origin.column.writetime.names tw_col2,w_col3 24 | -------------------------------------------------------------------------------- /SIT/smoke/03_ttl_writetime/t.out: -------------------------------------------------------------------------------- 1 | 2 | key|t_col1|tw_col2|w_col3|col4|wt_col4|ttl_col 3 | ---------+--------+---------+--------+------+- 4 | record3|AAAA|BBBB|CCCC|DDDD|1087383720000000|4 5 | record2|AAAA|BBBB|CCCC|DDDD|1087383720000000|4 6 | record1|AAAA|BBBB|CCCC|DDDD|1087383720000000|4 7 | 8 | (3rows) 9 | -------------------------------------------------------------------------------- /SIT/smoke/04_counters/breakData.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | Unless required by applicable law or agreed to in writing, software 7 | distributed under the License is distributed on an "AS IS" BASIS, 8 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | See the License for the specific language governing permissions and 10 | limitations under the License. 11 | */ 12 | 13 | UPDATE origin.smoke_counters SET col1=col1+1 WHERE key='record2'; 14 | 15 | -- CDM-29 discovered that this record will "zombie"... 16 | -- fixData will not correct it, fixForce will 17 | DELETE FROM target.smoke_counters where key='record3'; 18 | 19 | -- change by same amount on both sides, records should match 20 | UPDATE origin.smoke_counters SET col2=col2+1 WHERE key='record4'; 21 | UPDATE target.smoke_counters SET col2=col2+1 WHERE key='record4'; 22 | 23 | -- change by different amounts, they should be brought into alignment with 24 | -- origin, even if that means going down 25 | UPDATE origin.smoke_counters SET col2=col2+1 WHERE key='record5'; 26 | UPDATE target.smoke_counters SET col2=col2+2 WHERE key='record5'; 27 | 28 | SELECT * FROM origin.smoke_counters; 29 | SELECT * FROM target.smoke_counters; 30 | 31 | -------------------------------------------------------------------------------- /SIT/smoke/04_counters/cdm.fixData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 7 2 | Mismatch Record Count: 2 3 | Corrected Mismatch Record Count: 2 4 | Missing Record Count: 1 5 | Corrected Missing Record Count: 0 6 | Valid Record Count: 4 7 | Skipped Record Count: 0 8 | -------------------------------------------------------------------------------- /SIT/smoke/04_counters/cdm.fixForce.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 7 2 | Mismatch Record Count: 0 3 | Corrected Mismatch Record Count: 0 4 | Missing Record Count: 1 5 | Corrected Missing Record Count: 1 6 | Valid Record Count: 6 7 | Skipped Record Count: 0 8 | Error Record Count: 0 9 | -------------------------------------------------------------------------------- /SIT/smoke/04_counters/cdm.migrateData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 7 2 | Skipped Record Count: 0 3 | Write Record Count: 7 4 | Error Record Count: 0 5 | -------------------------------------------------------------------------------- /SIT/smoke/04_counters/cdm.txt: -------------------------------------------------------------------------------- 1 | migrateData com.datastax.cdm.job.Migrate migrate.properties 2 | validateData com.datastax.cdm.job.DiffData migrate.properties 3 | fixData com.datastax.cdm.job.DiffData fix.properties 4 | fixForce com.datastax.cdm.job.DiffData fix-force.properties 5 | -------------------------------------------------------------------------------- /SIT/smoke/04_counters/cdm.validateData.assert: -------------------------------------------------------------------------------- 1 | Read Record Count: 7 2 | Mismatch Record Count: 0 3 | Corrected Mismatch Record Count: 0 4 | Missing Record Count: 0 5 | Corrected Missing Record Count: 0 6 | Valid Record Count: 7 7 | Skipped Record Count: 0 8 | Error Record Count: 0 9 | -------------------------------------------------------------------------------- /SIT/smoke/04_counters/execute.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | #!/bin/bash -e 16 | 17 | workingDir="$1" 18 | cd "$workingDir" 19 | 20 | /local/cdm.sh -f cdm.txt -s migrateData -d "$workingDir" > cdm.migrateData.out 2>cdm.migrateData.err 21 | /local/cdm-assert.sh -f cdm.migrateData.out -a cdm.migrateData.assert -d "$workingDir" 22 | 23 | /local/cdm.sh -f cdm.txt -s validateData -d "$workingDir" > cdm.validateData.out 2>cdm.validateData.err 24 | /local/cdm-assert.sh -f cdm.validateData.out -a cdm.validateData.assert -d "$workingDir" 25 | 26 | cqlsh -u $CASS_USERNAME -p $CASS_PASSWORD $CASS_CLUSTER -f $workingDir/breakData.cql > $workingDir/breakData.out 2> $workingDir/breakData.err 27 | 28 | /local/cdm.sh -f cdm.txt -s fixData -d "$workingDir" > cdm.fixData.out 2>cdm.fixData.err 29 | /local/cdm-assert.sh -f cdm.fixData.out -a cdm.fixData.assert -d "$workingDir" 30 | 31 | /local/cdm.sh -f cdm.txt -s fixForce -d "$workingDir" > cdm.fixForce.out 2>cdm.fixForce.err 32 | /local/cdm-assert.sh -f cdm.fixForce.out -a cdm.fixForce.assert -d "$workingDir" 33 | -------------------------------------------------------------------------------- /SIT/smoke/04_counters/expected.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | Unless required by applicable law or agreed to in writing, software 7 | distributed under the License is distributed on an "AS IS" BASIS, 8 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | See the License for the specific language governing permissions and 10 | limitations under the License. 11 | */ 12 | 13 | SELECT key,col1,col2 FROM target.smoke_counters; 14 | -------------------------------------------------------------------------------- /SIT/smoke/04_counters/expected.out: -------------------------------------------------------------------------------- 1 | 2 | key | col1 | col2 3 | ---------+-------+------- 4 | record3 | 2000 | 4000 5 | record5 | 500 | 501 6 | record2 | 11 | 20 7 | record6 | 500 | null 8 | record1 | 1 | 2 9 | record7 | 1 | 2 10 | record4 | 10000 | 20001 11 | 12 | (7 rows) 13 | -------------------------------------------------------------------------------- /SIT/smoke/04_counters/fix-force.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | spark.cdm.connect.origin.host cdm-sit-cass 16 | spark.cdm.connect.target.host cdm-sit-cass 17 | 18 | spark.cdm.schema.origin.keyspaceTable origin.smoke_counters 19 | spark.cdm.schema.target.keyspaceTable target.smoke_counters 20 | spark.cdm.perfops.numParts 1 21 | 22 | spark.cdm.autocorrect.missing true 23 | spark.cdm.autocorrect.mismatch true 24 | spark.cdm.autocorrect.missing.counter true 25 | -------------------------------------------------------------------------------- /SIT/smoke/04_counters/fix.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | spark.cdm.connect.origin.host cdm-sit-cass 16 | spark.cdm.connect.target.host cdm-sit-cass 17 | 18 | spark.cdm.schema.origin.keyspaceTable origin.smoke_counters 19 | spark.cdm.schema.target.keyspaceTable target.smoke_counters 20 | spark.cdm.perfops.numParts 1 21 | 22 | spark.cdm.autocorrect.missing true 23 | spark.cdm.autocorrect.mismatch true 24 | -------------------------------------------------------------------------------- /SIT/smoke/04_counters/migrate.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | spark.cdm.connect.origin.host cdm-sit-cass 16 | spark.cdm.connect.target.host cdm-sit-cass 17 | 18 | spark.cdm.schema.origin.keyspaceTable origin.smoke_counters 19 | spark.cdm.schema.target.keyspaceTable target.smoke_counters 20 | spark.cdm.perfops.numParts 1 21 | 22 | -------------------------------------------------------------------------------- /SIT/smoke/04_counters/setup.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | Unless required by applicable law or agreed to in writing, software 7 | distributed under the License is distributed on an "AS IS" BASIS, 8 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | See the License for the specific language governing permissions and 10 | limitations under the License. 11 | */ 12 | 13 | DROP TABLE IF EXISTS origin.smoke_counters; 14 | CREATE TABLE origin.smoke_counters(key text,col1 counter,col2 counter, PRIMARY KEY (key)); 15 | 16 | UPDATE origin.smoke_counters set col1=col1+1, col2=col2+2 where key='record1'; 17 | UPDATE origin.smoke_counters set col1=col1+10, col2=col2+20 where key='record2'; 18 | UPDATE origin.smoke_counters set col1=col1+1000, col2=col2+2000 where key='record3'; 19 | UPDATE origin.smoke_counters set col1=col1+10000, col2=col2+20000 where key='record4'; 20 | UPDATE origin.smoke_counters set col1=col1+500, col2=col2+500 where key='record5'; 21 | UPDATE origin.smoke_counters set col1=col1+500 where key='record6'; 22 | UPDATE origin.smoke_counters set col1=col1+1, col2=col2+2 where key='record7'; 23 | 24 | DROP TABLE IF EXISTS target.smoke_counters; 25 | CREATE TABLE target.smoke_counters(key text,col1 counter,col2 counter, PRIMARY KEY (key)); 26 | 27 | UPDATE target.smoke_counters set col1=col1+1 where key='record7'; 28 | 29 | SELECT * FROM origin.smoke_counters; 30 | -------------------------------------------------------------------------------- /SIT/smoke/05_reserved_keyword/cdm.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | #!/bin/bash -e 16 | 17 | cat < "cdm.$scenario.out" 2>cdm.$scenario.err 22 | /local/cdm-assert.sh -f "cdm.$scenario.out" -a "cdm.$scenario.assert" -d "$workingDir" 23 | done 24 | -------------------------------------------------------------------------------- /SIT/smoke/06_vector/expected.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | SELECT * FROM target.smoke_vector; 16 | -------------------------------------------------------------------------------- /SIT/smoke/06_vector/expected.out: -------------------------------------------------------------------------------- 1 | 2 | col1 | embedding 3 | ------+-------------------- 4 | 1 | [1.1, 2.2, 3.3] 5 | 2 | [4.4, -0.01, 0.99] 6 | 7 | (2 rows) 8 | -------------------------------------------------------------------------------- /SIT/smoke/06_vector/migrate.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | spark.cdm.connect.origin.host cdm-sit-cass 16 | spark.cdm.connect.target.host cdm-sit-cass 17 | 18 | spark.cdm.schema.origin.keyspaceTable origin.smoke_vector 19 | spark.cdm.schema.target.keyspaceTable target.smoke_vector 20 | spark.cdm.perfops.numParts 1 21 | 22 | spark.cdm.autocorrect.missing true 23 | spark.cdm.autocorrect.mismatch true 24 | 25 | -------------------------------------------------------------------------------- /SIT/smoke/06_vector/setup.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); you 3 | may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | DROP TABLE IF EXISTS origin.smoke_vector; 16 | CREATE TABLE origin.smoke_vector (col1 int PRIMARY KEY, embedding vector); 17 | INSERT INTO origin.smoke_vector(col1, embedding) VALUES (1,[1.1,2.2,3.3]); 18 | INSERT INTO origin.smoke_vector(col1, embedding) VALUES (2,[4.4,-0.01,0.99]); 19 | 20 | DROP TABLE IF EXISTS target.smoke_vector; 21 | CREATE TABLE target.smoke_vector (col1 int PRIMARY KEY, embedding vector); 22 | -------------------------------------------------------------------------------- /rat-excludes.txt: -------------------------------------------------------------------------------- 1 | README.md 2 | rat-excludes.txt 3 | pom.xml 4 | CODEOWNERS 5 | CODE_OF_CONDUCT.md 6 | CONTRIBUTING.md 7 | RELEASE.md 8 | LICENSE.md 9 | Dockerfile 10 | .all-contributorsrc 11 | .snyk 12 | .snyk.ignore.example 13 | .github/** 14 | scripts/* 15 | test-backup/feature/* 16 | src/resources/log4j.xml 17 | SIT/*/*/*.assert 18 | SIT/*/*/*.out 19 | SIT/*/*/*.txt 20 | SIT/*/*/*.csv 21 | PERF/*/*.log 22 | toolchains.xml 23 | settings.xml 24 | SIT/common.sh 25 | SIT/environment.sh 26 | SIT/Makefile 27 | SIT/cdm-assert.sh 28 | SIT/cdm.sh 29 | SIT/test.sh 30 | PERF/cdm-v3.properties 31 | PERF/perf-iot.yaml 32 | PERF/testing.txt 33 | PERF/cdm-v4.properties 34 | PERF/cgpt.yaml 35 | PERF/cql-iot.yaml 36 | -------------------------------------------------------------------------------- /scripts/bump_minor_version: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | #!/bin/bash 16 | set -x 17 | CURRENT_VERSION=$(mvn help:evaluate -Dexpression=revision -q -DforceStdout) 18 | RELEASE_VERSION=${CURRENT_VERSION%%} 19 | BASE_VERSION=$(echo "$RELEASE_VERSION" | cut -d'.' -f1-2) 20 | MINOR_VERSION=$(echo "$RELEASE_VERSION" | cut -d'.' -f3) 21 | NEXT_MINOR_VERSION=$(( $MINOR_VERSION + 1 )) 22 | NEXT_SNAPSHOT="${BASE_VERSION}.${NEXT_MINOR_VERSION}" 23 | printf "next-snapshot: %s\n" "${NEXT_SNAPSHOT}" 24 | mvn versions:set-property -DallowSnapshots -DnewVersion="${NEXT_SNAPSHOT}" -Dproperty=revision 25 | -------------------------------------------------------------------------------- /scripts/get-latest-maven-version.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | #!/bin/bash 16 | 17 | MAVEN_BASE_VERSION=3.9 18 | MAVEN_REPO_URL="https://archive.apache.org/dist/maven/maven-3/" 19 | 20 | curl -sSL ${MAVEN_REPO_URL} | \ 21 | grep -o "${MAVEN_BASE_VERSION}\.[0-99]*\/" | \ 22 | sort -Vu | \ 23 | tail -n1 | \ 24 | sed 's/\///' 25 | -------------------------------------------------------------------------------- /src/main/java/com/datastax/cdm/cql/codec/AbstractBaseCodec.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.cql.codec; 17 | 18 | import com.datastax.cdm.properties.PropertyHelper; 19 | import com.datastax.oss.driver.api.core.type.codec.TypeCodec; 20 | 21 | public abstract class AbstractBaseCodec implements TypeCodec { 22 | private final PropertyHelper propertyHelper; 23 | 24 | public AbstractBaseCodec(PropertyHelper propertyHelper) { 25 | this.propertyHelper = propertyHelper; 26 | } 27 | 28 | } -------------------------------------------------------------------------------- /src/main/java/com/datastax/cdm/cql/codec/BLOB_TEXTCodec.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.cql.codec; 17 | 18 | import java.nio.ByteBuffer; 19 | 20 | import org.jetbrains.annotations.NotNull; 21 | 22 | import com.datastax.cdm.properties.PropertyHelper; 23 | import com.datastax.oss.driver.api.core.ProtocolVersion; 24 | import com.datastax.oss.driver.api.core.type.DataType; 25 | import com.datastax.oss.driver.api.core.type.DataTypes; 26 | import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; 27 | import com.datastax.oss.driver.api.core.type.reflect.GenericType; 28 | 29 | public class BLOB_TEXTCodec extends AbstractBaseCodec { 30 | 31 | public BLOB_TEXTCodec(PropertyHelper propertyHelper) { 32 | super(propertyHelper); 33 | } 34 | 35 | @Override 36 | public @NotNull GenericType getJavaType() { 37 | return GenericType.STRING; 38 | } 39 | 40 | @Override 41 | public @NotNull DataType getCqlType() { 42 | return DataTypes.BLOB; 43 | } 44 | 45 | @Override 46 | public ByteBuffer encode(String value, @NotNull ProtocolVersion protocolVersion) { 47 | if (value == null) { 48 | return null; 49 | } else { 50 | return TypeCodecs.BLOB.encode(ByteBuffer.wrap(value.getBytes()), protocolVersion); 51 | } 52 | } 53 | 54 | @Override 55 | public String decode(ByteBuffer bytes, @NotNull ProtocolVersion protocolVersion) { 56 | return TypeCodecs.TEXT.decode(bytes, protocolVersion); 57 | } 58 | 59 | @Override 60 | public @NotNull String format(String value) { 61 | ByteBuffer bb = ByteBuffer.wrap(value.getBytes()); 62 | return TypeCodecs.BLOB.format(bb); 63 | } 64 | 65 | @Override 66 | public String parse(String value) { 67 | ByteBuffer bb = TypeCodecs.BLOB.parse(value); 68 | return bb == null ? null : bb.toString(); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/com/datastax/cdm/cql/codec/BigInteger_BIGINTCodec.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.cql.codec; 17 | 18 | import java.nio.ByteBuffer; 19 | 20 | import org.jetbrains.annotations.NotNull; 21 | 22 | import com.datastax.cdm.properties.PropertyHelper; 23 | import com.datastax.oss.driver.api.core.ProtocolVersion; 24 | import com.datastax.oss.driver.api.core.type.DataType; 25 | import com.datastax.oss.driver.api.core.type.DataTypes; 26 | import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; 27 | import com.datastax.oss.driver.api.core.type.reflect.GenericType; 28 | 29 | public class BigInteger_BIGINTCodec extends AbstractBaseCodec { 30 | 31 | public BigInteger_BIGINTCodec(PropertyHelper propertyHelper) { 32 | super(propertyHelper); 33 | } 34 | 35 | @Override 36 | public @NotNull GenericType getJavaType() { 37 | return GenericType.INTEGER; 38 | } 39 | 40 | @Override 41 | public @NotNull DataType getCqlType() { 42 | return DataTypes.INT; 43 | } 44 | 45 | @Override 46 | public ByteBuffer encode(Integer value, @NotNull ProtocolVersion protocolVersion) { 47 | if (value == null) { 48 | return null; 49 | } else { 50 | return TypeCodecs.INT.encode(value, protocolVersion); 51 | } 52 | } 53 | 54 | @Override 55 | public Integer decode(ByteBuffer bytes, @NotNull ProtocolVersion protocolVersion) { 56 | return TypeCodecs.INT.decode(bytes, protocolVersion); 57 | } 58 | 59 | @Override 60 | public @NotNull String format(Integer value) { 61 | return TypeCodecs.INT.format(value); 62 | } 63 | 64 | @Override 65 | public Integer parse(String value) { 66 | return TypeCodecs.INT.parse(value); 67 | } 68 | 69 | } 70 | -------------------------------------------------------------------------------- /src/main/java/com/datastax/cdm/cql/codec/Codecset.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.cql.codec; 17 | 18 | public enum Codecset { 19 | INT_STRING, DOUBLE_STRING, BIGINT_STRING, BIGINT_BIGINTEGER, DECIMAL_STRING, TIMESTAMP_STRING_MILLIS, 20 | TIMESTAMP_STRING_FORMAT, POINT_TYPE, POLYGON_TYPE, DATE_RANGE, LINE_STRING, STRING_BLOB, ASCII_BLOB 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/com/datastax/cdm/cql/codec/TEXT_LongCodec.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.cql.codec; 17 | 18 | import java.nio.ByteBuffer; 19 | 20 | import org.jetbrains.annotations.NotNull; 21 | 22 | import com.datastax.cdm.properties.PropertyHelper; 23 | import com.datastax.oss.driver.api.core.ProtocolVersion; 24 | import com.datastax.oss.driver.api.core.type.DataType; 25 | import com.datastax.oss.driver.api.core.type.DataTypes; 26 | import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; 27 | import com.datastax.oss.driver.api.core.type.reflect.GenericType; 28 | 29 | public class TEXT_LongCodec extends AbstractBaseCodec { 30 | 31 | public TEXT_LongCodec(PropertyHelper propertyHelper) { 32 | super(propertyHelper); 33 | } 34 | 35 | @Override 36 | public @NotNull GenericType getJavaType() { 37 | return GenericType.LONG; 38 | } 39 | 40 | @Override 41 | public @NotNull DataType getCqlType() { 42 | return DataTypes.TEXT; 43 | } 44 | 45 | @Override 46 | public ByteBuffer encode(Long value, @NotNull ProtocolVersion protocolVersion) { 47 | if (value == null) { 48 | return null; 49 | } else { 50 | String stringValue = Long.toString(value); 51 | return TypeCodecs.TEXT.encode(stringValue, protocolVersion); 52 | } 53 | } 54 | 55 | @Override 56 | public Long decode(ByteBuffer bytes, @NotNull ProtocolVersion protocolVersion) { 57 | String stringValue = TypeCodecs.TEXT.decode(bytes, protocolVersion); 58 | return Long.parseLong(stringValue); 59 | } 60 | 61 | @Override 62 | public @NotNull String format(Long value) { 63 | return TypeCodecs.BIGINT.format(value); 64 | } 65 | 66 | @Override 67 | public Long parse(String value) { 68 | return value == null ? null : Long.parseLong(value); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/com/datastax/cdm/cql/statement/BaseCdmStatement.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.cql.statement; 17 | 18 | import java.util.ArrayList; 19 | import java.util.List; 20 | 21 | import com.datastax.cdm.cql.EnhancedSession; 22 | import com.datastax.cdm.properties.IPropertyHelper; 23 | import com.datastax.cdm.schema.CqlTable; 24 | import com.datastax.oss.driver.api.core.cql.PreparedStatement; 25 | 26 | public class BaseCdmStatement { 27 | 28 | protected IPropertyHelper propertyHelper; 29 | protected CqlTable cqlTable; 30 | protected String statement = ""; 31 | protected EnhancedSession session; 32 | 33 | protected List resultColumns = new ArrayList<>(); 34 | 35 | public BaseCdmStatement(IPropertyHelper propertyHelper, EnhancedSession session) { 36 | if (null == propertyHelper || null == session || null == session.getCqlTable()) 37 | throw new RuntimeException("PropertyHelper or EnhancedSession or EnhancedSession.getCqlTable() is not set"); 38 | this.propertyHelper = propertyHelper; 39 | this.cqlTable = session.getCqlTable(); 40 | this.session = session; 41 | } 42 | 43 | public PreparedStatement prepareStatement() { 44 | if (null == session || null == session.getCqlSession()) 45 | throw new RuntimeException("Session is not set"); 46 | if (null == statement || statement.isEmpty()) 47 | throw new RuntimeException("Statement is not set"); 48 | return session.getCqlSession().prepare(statement); 49 | } 50 | 51 | public String getCQL() { 52 | return statement; 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/com/datastax/cdm/feature/AbstractFeature.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.feature; 17 | 18 | import com.datastax.cdm.schema.CqlTable; 19 | 20 | public abstract class AbstractFeature implements Feature { 21 | 22 | protected boolean isEnabled = false; 23 | protected boolean isValid = true; 24 | protected boolean isLoaded = false; 25 | 26 | public AbstractFeature() { 27 | } 28 | 29 | @Override 30 | public boolean isEnabled() { 31 | if (!isLoaded) 32 | throw new RuntimeException("Feature not initialized"); 33 | return isEnabled; 34 | } 35 | 36 | @Override 37 | public boolean initializeAndValidate(CqlTable originTable, CqlTable targetTable) { 38 | if (!isLoaded) 39 | throw new RuntimeException("Feature not initialized"); 40 | if (!validateProperties()) { 41 | isEnabled = false; 42 | return false; 43 | } 44 | return isValid; 45 | } 46 | 47 | /** 48 | * Validate the properties of the feature typically called by loadProperties as well as initializeAndValidate. It 49 | * should set isValid to false if any properties are invalid, and ideally uses logger to inform the user of the 50 | * problems found in property configuration. 51 | * 52 | * @return true if the properties are valid, false otherwise 53 | */ 54 | protected boolean validateProperties() { 55 | return isValid; 56 | } 57 | 58 | @Override 59 | public String toString() { 60 | return String.format("%s{loaded:%s/valid:%s/enabled:%s}", this.getClass().getSimpleName(), isLoaded, isValid, 61 | isEnabled); 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /src/main/java/com/datastax/cdm/feature/Feature.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.feature; 17 | 18 | import com.datastax.cdm.properties.IPropertyHelper; 19 | import com.datastax.cdm.schema.CqlTable; 20 | 21 | public interface Feature { 22 | 23 | /** 24 | * Initializes the feature based on properties 25 | * 26 | * @param propertyHelper 27 | * propertyHelper containing initialized properties 28 | * 29 | * @return true if the properties appear to be valid, false otherwise 30 | */ 31 | public boolean loadProperties(IPropertyHelper propertyHelper); 32 | 33 | /** 34 | * Indicates if feature is enabled. 35 | * 36 | * @return true if the feature is enabled, false otherwise 37 | * 38 | * @throws RuntimeException 39 | * if the feature is not loaded 40 | */ 41 | public boolean isEnabled(); 42 | 43 | /** 44 | * Using the loaded properties, initializes the feature and validates it against the origin and target tables. This 45 | * method should be called after loadProperties() and before any other method. 46 | * 47 | * @param originCqlTable 48 | * origin CqlTable 49 | * @param targetCqlTable 50 | * target CqlTable 51 | * 52 | * @return true if the feature is valid, false otherwise 53 | * 54 | * @throws RuntimeException 55 | * if the feature is not loaded, or there is a problem with the properties relative to the tables. 56 | */ 57 | public boolean initializeAndValidate(CqlTable originCqlTable, CqlTable targetCqlTable); 58 | 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/com/datastax/cdm/feature/FeatureFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.feature; 17 | 18 | public class FeatureFactory { 19 | public static Feature getFeature(Featureset feature) { 20 | switch (feature) { 21 | case ORIGIN_FILTER: 22 | return new OriginFilterCondition(); 23 | case CONSTANT_COLUMNS: 24 | return new ConstantColumns(); 25 | case EXPLODE_MAP: 26 | return new ExplodeMap(); 27 | case EXTRACT_JSON: 28 | return new ExtractJson(); 29 | case WRITETIME_TTL: 30 | return new WritetimeTTL(); 31 | case GUARDRAIL_CHECK: 32 | return new Guardrail(); 33 | default: 34 | throw new IllegalArgumentException("Unknown feature: " + feature); 35 | } 36 | } 37 | 38 | public static Boolean isEnabled(Feature f) { 39 | return null != f && f.isEnabled(); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/com/datastax/cdm/feature/Featureset.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.feature; 17 | 18 | public enum Featureset { 19 | ORIGIN_FILTER, CONSTANT_COLUMNS, EXPLODE_MAP, EXTRACT_JSON, WRITETIME_TTL, GUARDRAIL_CHECK, 20 | TEST_UNIMPLEMENTED_FEATURE 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/com/datastax/cdm/job/CDMMetricsAccumulator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.job; 17 | 18 | import org.apache.spark.util.AccumulatorV2; 19 | 20 | import com.datastax.cdm.job.IJobSessionFactory.JobType; 21 | 22 | public class CDMMetricsAccumulator extends AccumulatorV2 { 23 | 24 | private static final long serialVersionUID = -4185304101452658315L; 25 | private JobCounter jobCounter; 26 | 27 | public CDMMetricsAccumulator(JobType jobType) { 28 | jobCounter = new JobCounter(jobType); 29 | } 30 | 31 | @Override 32 | public void add(JobCounter v) { 33 | jobCounter.add(v); 34 | } 35 | 36 | @Override 37 | public AccumulatorV2 copy() { 38 | return this; 39 | } 40 | 41 | @Override 42 | public boolean isZero() { 43 | return jobCounter.isZero(); 44 | } 45 | 46 | @Override 47 | public void merge(AccumulatorV2 other) { 48 | jobCounter.add(other.value()); 49 | } 50 | 51 | @Override 52 | public void reset() { 53 | jobCounter.reset(); 54 | } 55 | 56 | @Override 57 | public JobCounter value() { 58 | return jobCounter; 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/com/datastax/cdm/job/CopyJobSessionFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.job; 17 | 18 | import java.io.Serializable; 19 | 20 | import com.datastax.cdm.properties.PropertyHelper; 21 | import com.datastax.oss.driver.api.core.CqlSession; 22 | 23 | public class CopyJobSessionFactory implements IJobSessionFactory, Serializable { 24 | private static final long serialVersionUID = 5255029377029801421L; 25 | private static CopyJobSession jobSession = null; 26 | 27 | public AbstractJobSession getInstance(CqlSession originSession, CqlSession targetSession, 28 | PropertyHelper propHelper) { 29 | if (jobSession == null) { 30 | synchronized (CopyJobSession.class) { 31 | if (jobSession == null) { 32 | jobSession = new CopyJobSession(originSession, targetSession, propHelper); 33 | } 34 | } 35 | } 36 | return jobSession; 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/com/datastax/cdm/job/CounterUnit.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.job; 17 | 18 | import java.io.Serializable; 19 | 20 | public class CounterUnit implements Serializable { 21 | 22 | private static final long serialVersionUID = 2194336948011681878L; 23 | private long count = 0; 24 | private long interimCount = 0; 25 | 26 | public void increment(long incrementBy) { 27 | interimCount += incrementBy; 28 | } 29 | 30 | public long getInterimCount() { 31 | return interimCount; 32 | } 33 | 34 | public void reset() { 35 | interimCount = 0; 36 | } 37 | 38 | public void setCount(long value) { 39 | count = value; 40 | } 41 | 42 | public void addToCount() { 43 | count += interimCount; 44 | reset(); 45 | } 46 | 47 | public long getCount() { 48 | return count; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/com/datastax/cdm/job/DiffJobSessionFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.job; 17 | 18 | import java.io.Serializable; 19 | 20 | import com.datastax.cdm.properties.PropertyHelper; 21 | import com.datastax.oss.driver.api.core.CqlSession; 22 | 23 | public class DiffJobSessionFactory implements IJobSessionFactory, Serializable { 24 | private static final long serialVersionUID = -3543616512495020278L; 25 | private static DiffJobSession jobSession = null; 26 | 27 | public AbstractJobSession getInstance(CqlSession originSession, CqlSession targetSession, 28 | PropertyHelper propHelper) { 29 | if (jobSession == null) { 30 | synchronized (DiffJobSession.class) { 31 | if (jobSession == null) { 32 | jobSession = new DiffJobSession(originSession, targetSession, propHelper); 33 | } 34 | } 35 | } 36 | return jobSession; 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/com/datastax/cdm/job/GuardrailCheckJobSessionFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.job; 17 | 18 | import java.io.Serializable; 19 | 20 | import com.datastax.cdm.properties.PropertyHelper; 21 | import com.datastax.oss.driver.api.core.CqlSession; 22 | 23 | public class GuardrailCheckJobSessionFactory implements IJobSessionFactory, Serializable { 24 | private static final long serialVersionUID = -4673384128807660843L; 25 | private static GuardrailCheckJobSession jobSession = null; 26 | 27 | public AbstractJobSession getInstance(CqlSession originSession, CqlSession targetSession, 28 | PropertyHelper propHelper) { 29 | if (jobSession == null) { 30 | synchronized (GuardrailCheckJobSession.class) { 31 | if (jobSession == null) { 32 | jobSession = new GuardrailCheckJobSession(originSession, targetSession, propHelper); 33 | } 34 | } 35 | } 36 | return jobSession; 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/com/datastax/cdm/job/IJobSessionFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.job; 17 | 18 | import com.datastax.cdm.properties.PropertyHelper; 19 | import com.datastax.oss.driver.api.core.CqlSession; 20 | 21 | public interface IJobSessionFactory { 22 | public enum JobType { 23 | MIGRATE, VALIDATE, GUARDRAIL 24 | } 25 | 26 | AbstractJobSession getInstance(CqlSession originSession, CqlSession targetSession, PropertyHelper propHelper); 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/datastax/cdm/job/PartitionRange.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.job; 17 | 18 | import java.io.Serializable; 19 | import java.math.BigInteger; 20 | 21 | import com.datastax.cdm.job.IJobSessionFactory.JobType; 22 | 23 | public class PartitionRange implements Serializable { 24 | private static final long serialVersionUID = 1L; 25 | 26 | private final BigInteger min; 27 | private final BigInteger max; 28 | protected JobCounter jobCounter; 29 | 30 | public PartitionRange(BigInteger min, BigInteger max, JobType jobType) { 31 | this.min = min; 32 | this.max = max; 33 | jobCounter = new JobCounter(jobType); 34 | } 35 | 36 | public BigInteger getMin() { 37 | return min; 38 | } 39 | 40 | public BigInteger getMax() { 41 | return max; 42 | } 43 | 44 | public JobCounter getJobCounter() { 45 | return jobCounter; 46 | } 47 | 48 | public String toString() { 49 | return "Processing partition for token range " + min + " to " + max; 50 | } 51 | } -------------------------------------------------------------------------------- /src/main/java/com/datastax/cdm/job/RunNotStartedException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.job; 17 | 18 | public class RunNotStartedException extends Exception { 19 | 20 | private static final long serialVersionUID = -4108800389847708120L; 21 | 22 | public RunNotStartedException(String message) { 23 | super(message); 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/com/datastax/cdm/properties/IPropertyHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.properties; 17 | 18 | import java.util.List; 19 | 20 | import org.apache.spark.SparkConf; 21 | 22 | public interface IPropertyHelper { 23 | 24 | void initializeSparkConf(SparkConf sc); 25 | 26 | Object setProperty(String propertyName, Object propertyValue); 27 | 28 | String getString(String propertyName); 29 | 30 | List getStringList(String propertyName); 31 | 32 | Number getNumber(String propertyName); 33 | 34 | Integer getInteger(String propertyName); 35 | 36 | Long getLong(String propertyName); 37 | 38 | List getNumberList(String propertyName); 39 | 40 | List getIntegerList(String propertyName); 41 | 42 | Boolean getBoolean(String propertyName); 43 | 44 | String getAsString(String propertyName); 45 | 46 | boolean isSparkConfFullyLoaded(); 47 | 48 | boolean meetsMinimum(String valueName, Long testValue, Long minimumValue); 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/com/datastax/cdm/schema/Table.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.schema; 17 | 18 | import java.util.List; 19 | 20 | import com.datastax.oss.driver.api.core.type.DataType; 21 | 22 | public interface Table { 23 | String getKeyspaceName(); 24 | 25 | String getTableName(); 26 | 27 | List getColumnNames(boolean asCql); 28 | 29 | List getColumnCqlTypes(); 30 | } 31 | -------------------------------------------------------------------------------- /src/main/scala/com/datastax/cdm/job/BasePartitionJob.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.job 17 | 18 | import java.util 19 | import scala.reflect.io.File 20 | import com.datastax.cdm.feature.TrackRun 21 | import com.datastax.cdm.properties.KnownProperties 22 | 23 | abstract class BasePartitionJob extends BaseJob[PartitionRange] { 24 | var trackRunFeature: TrackRun = _ 25 | var keyspaceTableValue: String = _ 26 | 27 | override def getParts(pieces: Int): util.Collection[PartitionRange] = { 28 | var keyspaceTable: Option[String] = Option(propertyHelper.getString(KnownProperties.TARGET_KEYSPACE_TABLE)) 29 | .filter(_.nonEmpty) 30 | .orElse(Option(propertyHelper.getString(KnownProperties.ORIGIN_KEYSPACE_TABLE))) 31 | 32 | keyspaceTableValue = keyspaceTable.getOrElse { 33 | throw new RuntimeException("Both " + KnownProperties.TARGET_KEYSPACE_TABLE + " and " 34 | + KnownProperties.ORIGIN_KEYSPACE_TABLE + " properties are missing.") 35 | } 36 | 37 | if (trackRun) { 38 | trackRunFeature = targetConnection.withSessionDo(targetSession => new TrackRun(targetSession, keyspaceTableValue)) 39 | } 40 | 41 | if (prevRunId != 0) { 42 | try { 43 | trackRunFeature.getPendingPartitions(prevRunId, jobType) 44 | } catch { 45 | case e: RunNotStartedException => { 46 | abstractLogger.warn(e.getMessage) 47 | SplitPartitions.getRandomSubPartitions(pieces, minPartition, maxPartition, coveragePercent, jobType) 48 | } 49 | } 50 | } else { 51 | SplitPartitions.getRandomSubPartitions(pieces, minPartition, maxPartition, coveragePercent, jobType) 52 | } 53 | } 54 | 55 | } -------------------------------------------------------------------------------- /src/main/scala/com/datastax/cdm/job/ConnectionDetails.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.job 17 | 18 | case class ConnectionDetails( 19 | scbPath: String, 20 | host: String, 21 | port: String, 22 | username: String, 23 | password: String, 24 | sslEnabled: String, 25 | trustStorePath: String, 26 | trustStorePassword: String, 27 | trustStoreType: String, 28 | keyStorePath: String, 29 | keyStorePassword: String, 30 | enabledAlgorithms: String, 31 | isAstra: Boolean 32 | ) -------------------------------------------------------------------------------- /src/main/scala/com/datastax/cdm/job/GuardrailCheck.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.job 17 | 18 | import com.datastax.cdm.data.PKFactory.Side 19 | import com.datastax.cdm.properties.KnownProperties 20 | import com.datastax.cdm.job.IJobSessionFactory.JobType 21 | 22 | object GuardrailCheck extends BasePartitionJob { 23 | jobType = JobType.GUARDRAIL 24 | setup("Guardrail Check Job", new GuardrailCheckJobSessionFactory()) 25 | execute() 26 | finish() 27 | 28 | protected def execute(): Unit = { 29 | if (!parts.isEmpty()) { 30 | originConnection.withSessionDo(originSession => 31 | jobFactory.getInstance(originSession, null, propertyHelper)); 32 | var ma = new CDMMetricsAccumulator(jobType) 33 | sContext.register(ma, "CDMMetricsAccumulator") 34 | 35 | val bcOriginConfig = sContext.broadcast(sContext.getConf) 36 | val bcConnectionFetcher = sContext.broadcast(connectionFetcher) 37 | val bcPropHelper = sContext.broadcast(propertyHelper) 38 | val bcJobFactory = sContext.broadcast(jobFactory) 39 | 40 | slices.foreach(slice => { 41 | if (null == originConnection) { 42 | originConnection = bcConnectionFetcher.value.getConnection(bcOriginConfig.value, Side.ORIGIN, bcPropHelper.value.getString(KnownProperties.READ_CL), 0) 43 | } 44 | originConnection.withSessionDo(originSession => 45 | bcJobFactory.value.getInstance(originSession, null, bcPropHelper.value) 46 | .processPartitionRange(slice, null, runId)) 47 | ma.add(slice.getJobCounter()) 48 | }) 49 | 50 | ma.value.printMetrics(0, null); 51 | } 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /src/resources/log4j2.properties: -------------------------------------------------------------------------------- 1 | # Copyright DataStax, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | appender.0.type = Console 17 | appender.0.name = CONSOLE 18 | appender.0.layout.type = PatternLayout 19 | appender.0.layout.pattern = %d %-5p [%t] %c{1}:%L - %m%n 20 | 21 | appender.1.type = RollingFile 22 | appender.1.name = MAIN 23 | appender.1.fileName = cdm_logs/cdm.log 24 | appender.1.filePattern = cdm_logs/cdm.%d{yyyy-MM-dd-HHmm}.%i.log 25 | appender.1.layout.type = PatternLayout 26 | appender.1.layout.pattern = %d %-5p [%t] %c{1}:%L - %m%n 27 | appender.1.policy.type = Policies 28 | appender.1.policy.0.type = OnStartupTriggeringPolicy 29 | appender.1.policy.1.type = SizeBasedTriggeringPolicy 30 | appender.1.policy.1.size = 10m 31 | appender.1.strategy.type = DefaultRolloverStrategy 32 | appender.1.strategy.max = 100 33 | 34 | appender.2.type = RollingFile 35 | appender.2.name = DIFF 36 | appender.2.fileName = cdm_logs/cdm_diff.log 37 | appender.2.filePattern = cdm_logs/cdm_diff.%d{yyyy-MM-dd-HHmm}.%i.log 38 | appender.2.layout.type = PatternLayout 39 | appender.2.layout.pattern = %d %-5p [%t] %c{1}:%L - %m%n 40 | appender.2.policy.type = Policies 41 | appender.2.policy.0.type = OnStartupTriggeringPolicy 42 | appender.2.policy.1.type = SizeBasedTriggeringPolicy 43 | appender.2.policy.1.size = 10m 44 | appender.2.strategy.type = DefaultRolloverStrategy 45 | appender.2.strategy.max = 100 46 | 47 | rootLogger.level = INFO 48 | rootLogger.appenderRef.0.ref = CONSOLE 49 | rootLogger.appenderRef.0.level = INFO 50 | 51 | logger.0.name = com.datastax.cdm 52 | logger.0.level = INFO 53 | logger.0.additivity = false 54 | logger.0.appenderRef.0.ref = MAIN 55 | 56 | logger.1.name = com.datastax.cdm.job.DiffJobSession 57 | logger.1.level = ERROR 58 | logger.1.additivity = false 59 | logger.1.appenderRef.0.ref = DIFF 60 | -------------------------------------------------------------------------------- /src/resources/log4j2_docker.properties: -------------------------------------------------------------------------------- 1 | # Copyright DataStax, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | appender.0.type = Console 17 | appender.0.name = CONSOLE 18 | appender.0.layout.type = PatternLayout 19 | appender.0.layout.pattern = %d %-5p [%t] %c{1}:%L - %m%n 20 | 21 | rootLogger.level = INFO 22 | rootLogger.appenderRef.0.ref = CONSOLE 23 | rootLogger.appenderRef.0.level = INFO 24 | -------------------------------------------------------------------------------- /src/test/java/com/datastax/cdm/cql/codec/ASCII_BLOBCodecTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.cql.codec; 17 | 18 | import static org.junit.jupiter.api.Assertions.assertEquals; 19 | 20 | import java.nio.ByteBuffer; 21 | 22 | import org.junit.jupiter.api.Assertions; 23 | import org.junit.jupiter.api.BeforeEach; 24 | import org.junit.jupiter.api.Test; 25 | 26 | import com.datastax.oss.driver.api.core.ProtocolVersion; 27 | import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; 28 | 29 | public class ASCII_BLOBCodecTest { 30 | private final String INPUT = "Encode this Text string to Blob"; 31 | 32 | private ASCII_BLOBCodec codec; 33 | 34 | @BeforeEach 35 | public void setup() { 36 | codec = new ASCII_BLOBCodec(null); 37 | } 38 | 39 | @Test 40 | public void encodeDecode() { 41 | ByteBuffer buffer = codec.encode(ByteBuffer.wrap(INPUT.getBytes()), ProtocolVersion.V4); 42 | ByteBuffer retBuffer = codec.decode(buffer, ProtocolVersion.V4); 43 | assertEquals("'" + INPUT + "'", codec.format(retBuffer)); 44 | assertEquals(retBuffer, codec.parse(INPUT)); 45 | } 46 | 47 | @Test 48 | void testFormat() { 49 | String expected = TypeCodecs.ASCII.format(INPUT); 50 | 51 | String result = codec.format(ByteBuffer.wrap(INPUT.getBytes())); 52 | Assertions.assertEquals(expected, result); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/test/java/com/datastax/cdm/cql/codec/BLOB_ASCIICodecTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.cql.codec; 17 | 18 | import static org.junit.jupiter.api.Assertions.assertEquals; 19 | 20 | import java.nio.ByteBuffer; 21 | 22 | import org.junit.jupiter.api.Assertions; 23 | import org.junit.jupiter.api.BeforeEach; 24 | import org.junit.jupiter.api.Test; 25 | 26 | import com.datastax.oss.driver.api.core.ProtocolVersion; 27 | import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; 28 | 29 | public class BLOB_ASCIICodecTest { 30 | private final String INPUT = "Encode this Text string to Blob"; 31 | 32 | private BLOB_ASCIICodec codec; 33 | 34 | @BeforeEach 35 | public void setup() { 36 | codec = new BLOB_ASCIICodec(null); 37 | } 38 | 39 | @Test 40 | public void testEncode() { 41 | ByteBuffer buffer = codec.encode(INPUT, ProtocolVersion.V4); 42 | String retBuffer = codec.decode(buffer, ProtocolVersion.V4); 43 | assertEquals(INPUT, retBuffer); 44 | } 45 | 46 | @Test 47 | void testFormat() { 48 | ByteBuffer bb = ByteBuffer.wrap(INPUT.getBytes()); 49 | String expected = TypeCodecs.BLOB.format(bb); 50 | 51 | String result = codec.format(INPUT); 52 | Assertions.assertEquals(expected, result); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/test/java/com/datastax/cdm/cql/codec/BLOB_TextCodecTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.cql.codec; 17 | 18 | import static org.junit.jupiter.api.Assertions.assertEquals; 19 | 20 | import java.nio.ByteBuffer; 21 | 22 | import org.junit.jupiter.api.Assertions; 23 | import org.junit.jupiter.api.BeforeEach; 24 | import org.junit.jupiter.api.Test; 25 | 26 | import com.datastax.oss.driver.api.core.ProtocolVersion; 27 | import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; 28 | 29 | public class BLOB_TextCodecTest { 30 | private final String INPUT = "Encode this Text string to Blob"; 31 | 32 | private BLOB_TEXTCodec codec; 33 | 34 | @BeforeEach 35 | public void setup() { 36 | codec = new BLOB_TEXTCodec(null); 37 | } 38 | 39 | @Test 40 | public void testEncode() { 41 | ByteBuffer buffer = codec.encode(INPUT, ProtocolVersion.V4); 42 | String retBuffer = codec.decode(buffer, ProtocolVersion.V4); 43 | assertEquals(INPUT, retBuffer); 44 | } 45 | 46 | @Test 47 | void testFormat() { 48 | ByteBuffer bb = ByteBuffer.wrap(INPUT.getBytes()); 49 | String expected = TypeCodecs.BLOB.format(bb); 50 | 51 | String result = codec.format(INPUT); 52 | Assertions.assertEquals(expected, result); 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/test/java/com/datastax/cdm/cql/codec/CodecTestHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.cql.codec; 17 | 18 | import java.nio.ByteBuffer; 19 | 20 | import org.junit.jupiter.api.Assertions; 21 | 22 | public class CodecTestHelper { 23 | public static void assertByteBufferEquals(ByteBuffer expected, ByteBuffer actual) { 24 | Assertions.assertEquals(expected.remaining(), actual.remaining(), 25 | () -> String.format( 26 | "ByteBuffers have different remaining bytes:%nExpected byte[]: %s%nActual byte[]: %s", 27 | byteBufferToHexString(expected), byteBufferToHexString(actual))); 28 | 29 | Assertions.assertTrue(expected.equals(actual), 30 | () -> String.format("ByteBuffers are not equal:%nExpected byte[]: %s%nActual byte[]: %s", 31 | byteBufferToHexString(expected), byteBufferToHexString(actual))); 32 | } 33 | 34 | private static String byteBufferToHexString(ByteBuffer buffer) { 35 | byte[] bytes = new byte[buffer.remaining()]; 36 | buffer.mark(); 37 | buffer.get(bytes); 38 | buffer.reset(); 39 | StringBuilder hexString = new StringBuilder(); 40 | for (byte b : bytes) { 41 | hexString.append(String.format("%02X ", b)); 42 | } 43 | return hexString.toString(); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/test/java/com/datastax/cdm/cql/codec/TEXT_BLOBCodecTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.cql.codec; 17 | 18 | import static org.junit.jupiter.api.Assertions.assertEquals; 19 | 20 | import java.nio.ByteBuffer; 21 | 22 | import org.junit.jupiter.api.BeforeEach; 23 | import org.junit.jupiter.api.Test; 24 | 25 | import com.datastax.oss.driver.api.core.ProtocolVersion; 26 | 27 | public class TEXT_BLOBCodecTest { 28 | private final String INPUT = "Encode this Text string to Blob"; 29 | 30 | private TEXT_BLOBCodec codec; 31 | 32 | @BeforeEach 33 | public void setup() { 34 | codec = new TEXT_BLOBCodec(null); 35 | } 36 | 37 | @Test 38 | public void encodeDecode() { 39 | ByteBuffer buffer = codec.encode(ByteBuffer.wrap(INPUT.getBytes()), ProtocolVersion.V4); 40 | ByteBuffer retBuffer = codec.decode(buffer, ProtocolVersion.V4); 41 | assertEquals("'" + INPUT + "'", codec.format(retBuffer)); 42 | assertEquals(retBuffer, codec.parse(INPUT)); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/test/java/com/datastax/cdm/data/MockitoExtension.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.data; 17 | 18 | import org.junit.jupiter.api.extension.Extension; 19 | 20 | public class MockitoExtension implements Extension { 21 | } 22 | -------------------------------------------------------------------------------- /src/test/java/com/datastax/cdm/data/RecordTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright DataStax, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.datastax.cdm.data; 17 | 18 | import static org.junit.jupiter.api.Assertions.assertEquals; 19 | import static org.junit.jupiter.api.Assertions.assertThrows; 20 | import static org.junit.jupiter.api.Assertions.assertTrue; 21 | 22 | import org.junit.jupiter.api.Test; 23 | 24 | public class RecordTest { 25 | 26 | @Test 27 | public void enumValues() { 28 | assertEquals(4, Record.Diff.values().length); 29 | } 30 | 31 | @Test 32 | public void recordException() { 33 | Exception e = assertThrows(IllegalArgumentException.class, () -> { 34 | new Record(null, null); 35 | }); 36 | assertTrue(e.getMessage().contains("pk and at least one row must be provided")); 37 | } 38 | 39 | @Test 40 | public void recordWithFutureRow() { 41 | Exception e = assertThrows(IllegalArgumentException.class, () -> { 42 | new Record(null, null, null); 43 | }); 44 | assertTrue(e.getMessage().contains("pk and at least one row must be provided")); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/test/java/com/datastax/cdm/job/SplitPartitionsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); 3 | you may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | 15 | package com.datastax.cdm.job; 16 | 17 | import static org.junit.jupiter.api.Assertions.assertEquals; 18 | 19 | import java.math.BigInteger; 20 | import java.util.List; 21 | 22 | import org.junit.jupiter.api.AfterEach; 23 | import org.junit.jupiter.api.Test; 24 | 25 | import com.datastax.cdm.job.IJobSessionFactory.JobType; 26 | import com.datastax.cdm.properties.PropertyHelper; 27 | 28 | public class SplitPartitionsTest { 29 | @AfterEach 30 | void tearDown() { 31 | PropertyHelper.destroyInstance(); 32 | } 33 | 34 | @Test 35 | void getRandomSubPartitionsTest() { 36 | List partitions = SplitPartitions.getRandomSubPartitions(10, BigInteger.ONE, 37 | BigInteger.valueOf(100), 100, JobType.MIGRATE); 38 | assertEquals(10, partitions.size()); 39 | partitions.forEach(p -> { 40 | assertEquals(9, p.getMax().longValue() - p.getMin().longValue()); 41 | }); 42 | } 43 | 44 | @Test 45 | void getRandomSubPartitionsTestOver100() { 46 | List partitions = SplitPartitions.getRandomSubPartitions(8, BigInteger.ONE, 47 | BigInteger.valueOf(44), 200, JobType.MIGRATE); 48 | assertEquals(8, partitions.size()); 49 | } 50 | 51 | } 52 | --------------------------------------------------------------------------------