├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE.md ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yml └── workflows │ ├── dependabot-auto-merge.yml │ ├── gradle-build-ci.yml │ ├── jacoco.yml │ ├── markdown-link-check.yml │ ├── new_ticket.yml │ ├── pr-lint.yml │ ├── repo-sync.yml │ └── trufflehog.yml ├── .gitignore ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── NOTICE ├── README.md ├── SECURITY.md ├── STYLE_GUIDELINES.md ├── SUPPORT.md ├── THIRD-PARTY-NOTICES ├── build.gradle ├── c3r-cli-spark ├── build.gradle └── src │ ├── integration-test │ └── spark_integration_tests.py │ ├── main │ ├── java │ │ └── com │ │ │ └── amazonaws │ │ │ └── c3r │ │ │ └── spark │ │ │ ├── action │ │ │ ├── SparkMarshaller.java │ │ │ ├── SparkUnmarshaller.java │ │ │ └── package-info.java │ │ │ ├── cli │ │ │ ├── CliDescriptions.java │ │ │ ├── DecryptMode.java │ │ │ ├── EncryptMode.java │ │ │ ├── Main.java │ │ │ ├── SchemaMode.java │ │ │ └── package-info.java │ │ │ ├── config │ │ │ ├── SparkConfig.java │ │ │ ├── SparkDecryptConfig.java │ │ │ ├── SparkEncryptConfig.java │ │ │ └── package-info.java │ │ │ ├── io │ │ │ ├── csv │ │ │ │ ├── Csv.java │ │ │ │ ├── CsvBatch.java │ │ │ │ ├── CsvBatchWrite.java │ │ │ │ ├── CsvDataWriter.java │ │ │ │ ├── CsvDataWriterFactory.java │ │ │ │ ├── CsvInputPartition.java │ │ │ │ ├── CsvPartitionReader.java │ │ │ │ ├── CsvPartitionReaderFactory.java │ │ │ │ ├── CsvScan.java │ │ │ │ ├── CsvScanBuilder.java │ │ │ │ ├── CsvTable.java │ │ │ │ ├── CsvWrite.java │ │ │ │ ├── CsvWriteBuilder.java │ │ │ │ ├── SchemaUtil.java │ │ │ │ ├── SparkCsvReader.java │ │ │ │ ├── SparkCsvWriter.java │ │ │ │ └── package-info.java │ │ │ ├── parquet │ │ │ │ ├── SparkParquetReader.java │ │ │ │ ├── SparkParquetWriter.java │ │ │ │ └── package-info.java │ │ │ └── schema │ │ │ │ ├── CsvSchemaGenerator.java │ │ │ │ ├── InteractiveSchemaGenerator.java │ │ │ │ ├── ParquetSchemaGenerator.java │ │ │ │ ├── SchemaGenerator.java │ │ │ │ ├── SchemaGeneratorUtils.java │ │ │ │ ├── TemplateSchemaGenerator.java │ │ │ │ └── package-info.java │ │ │ └── utils │ │ │ ├── C3rCliSparkProperties.java │ │ │ ├── SparkSessionUtil.java │ │ │ └── package-info.java │ └── resources │ │ └── log4j2.xml │ └── test │ ├── java │ └── com │ │ └── amazonaws │ │ └── c3r │ │ └── spark │ │ ├── action │ │ ├── SparkMarshallerTest.java │ │ └── SparkUnmarshallerTest.java │ │ ├── cleanrooms │ │ └── CleanRoomsDaoTestUtility.java │ │ ├── cli │ │ ├── CliTestUtility.java │ │ ├── DecryptCliConfigTestUtility.java │ │ ├── DecryptModeDryRunTest.java │ │ ├── EncryptCliConfigTestUtility.java │ │ ├── EncryptModeDryRunTest.java │ │ ├── MainArgParseTest.java │ │ ├── MainCsvSingleRowRoundTripTest.java │ │ ├── MainEnvVarKeyInvalidTest.java │ │ ├── MainEnvVarKeyValidTest.java │ │ ├── MainPerfTest.java │ │ ├── MainTest.java │ │ ├── SchemaCliConfigTestUtility.java │ │ ├── SchemaModeDryRunTest.java │ │ └── SchemaModeTest.java │ │ ├── config │ │ ├── SparkDecryptConfigTest.java │ │ └── SparkEncryptConfigTest.java │ │ ├── io │ │ ├── CsvTestUtility.java │ │ ├── ParquetTestUtility.java │ │ ├── csv │ │ │ ├── SparkCsvReaderTest.java │ │ │ └── SparkCsvWriterTest.java │ │ ├── parquet │ │ │ ├── SparkParquetReaderTest.java │ │ │ └── SparkParquetWriterTest.java │ │ └── schema │ │ │ ├── CsvSchemaGeneratorTest.java │ │ │ ├── InteractiveSchemaGeneratorTest.java │ │ │ ├── ParquetSchemaGeneratorTest.java │ │ │ └── TemplateSchemaGeneratorTest.java │ │ └── utils │ │ ├── DecryptSdkConfigTestUtility.java │ │ ├── EncryptSdkConfigTestUtility.java │ │ ├── FileTestUtility.java │ │ ├── GeneralTestUtility.java │ │ ├── SparkSessionTestUtility.java │ │ ├── StringTestUtility.java │ │ ├── StringTestUtilityTest.java │ │ ├── TableGeneratorTestUtility.java │ │ └── TimingResultTestUtility.java │ └── resources │ └── log4j2.xml ├── c3r-cli ├── build.gradle └── src │ ├── integration-test │ └── integration_tests.py │ ├── main │ ├── java │ │ └── com │ │ │ └── amazonaws │ │ │ └── c3r │ │ │ ├── cli │ │ │ ├── CliDescriptions.java │ │ │ ├── DecryptMode.java │ │ │ ├── EncryptMode.java │ │ │ ├── Main.java │ │ │ ├── SchemaMode.java │ │ │ └── package-info.java │ │ │ ├── io │ │ │ └── schema │ │ │ │ ├── CsvSchemaGenerator.java │ │ │ │ ├── InteractiveSchemaGenerator.java │ │ │ │ ├── ParquetSchemaGenerator.java │ │ │ │ ├── SchemaGenerator.java │ │ │ │ ├── SchemaGeneratorUtils.java │ │ │ │ ├── TemplateSchemaGenerator.java │ │ │ │ └── package-info.java │ │ │ └── utils │ │ │ ├── C3rCliProperties.java │ │ │ └── package-info.java │ └── resources │ │ └── log4j2.xml │ └── test │ ├── java │ └── com │ │ └── amazonaws │ │ └── c3r │ │ ├── cleanrooms │ │ └── CleanRoomsDaoTestUtility.java │ │ ├── cli │ │ ├── CliTestUtility.java │ │ ├── DecryptCliConfigTestUtility.java │ │ ├── DecryptModeDryRunTest.java │ │ ├── EncryptCliConfigTestUtility.java │ │ ├── EncryptModeDryRunTest.java │ │ ├── MainArgParseTest.java │ │ ├── MainCsvSingleRowRoundTripTest.java │ │ ├── MainEnvVarKeyInvalidTest.java │ │ ├── MainEnvVarKeyValidTest.java │ │ ├── MainErrorMessageTest.java │ │ ├── MainPerfTest.java │ │ ├── MainTest.java │ │ ├── SchemaCliConfigTestUtility.java │ │ ├── SchemaModeDryRunTest.java │ │ └── SchemaModeTest.java │ │ ├── io │ │ ├── CsvTestUtility.java │ │ ├── ParquetTestUtility.java │ │ └── schema │ │ │ ├── CsvSchemaGeneratorTest.java │ │ │ ├── InteractiveSchemaGeneratorTest.java │ │ │ ├── ParquetSchemaGeneratorTest.java │ │ │ └── TemplateSchemaGeneratorTest.java │ │ └── utils │ │ ├── FileTestUtility.java │ │ ├── GeneralTestUtility.java │ │ ├── StringTestUtility.java │ │ ├── StringTestUtilityTest.java │ │ ├── TableGeneratorTestUtility.java │ │ └── TimingResultTestUtility.java │ └── resources │ └── log4j2.xml ├── c3r-sdk-core ├── build.gradle └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── amazonaws │ │ │ └── c3r │ │ │ ├── CleartextTransformer.java │ │ │ ├── FingerprintTransformer.java │ │ │ ├── SealedTransformer.java │ │ │ ├── Transformer.java │ │ │ ├── action │ │ │ ├── CsvRowMarshaller.java │ │ │ ├── CsvRowUnmarshaller.java │ │ │ ├── RowMarshaller.java │ │ │ ├── RowUnmarshaller.java │ │ │ └── package-info.java │ │ │ ├── cleanrooms │ │ │ ├── CleanRoomsDao.java │ │ │ └── package-info.java │ │ │ ├── config │ │ │ ├── ClientSettings.java │ │ │ ├── ColumnHeader.java │ │ │ ├── ColumnInsight.java │ │ │ ├── ColumnSchema.java │ │ │ ├── ColumnType.java │ │ │ ├── Config.java │ │ │ ├── DecryptConfig.java │ │ │ ├── EncryptConfig.java │ │ │ ├── MappedTableSchema.java │ │ │ ├── Pad.java │ │ │ ├── PadType.java │ │ │ ├── PositionalTableSchema.java │ │ │ ├── SimpleFileConfig.java │ │ │ ├── TableSchema.java │ │ │ └── package-info.java │ │ │ ├── data │ │ │ ├── ClientDataInfo.java │ │ │ ├── ClientDataType.java │ │ │ ├── ClientValueWithMetadata.java │ │ │ ├── CsvRow.java │ │ │ ├── CsvValue.java │ │ │ ├── CsvValueFactory.java │ │ │ ├── Row.java │ │ │ ├── Units.java │ │ │ ├── Value.java │ │ │ ├── ValueConverter.java │ │ │ ├── ValueFactory.java │ │ │ └── package-info.java │ │ │ ├── encryption │ │ │ ├── EncryptionContext.java │ │ │ ├── Encryptor.java │ │ │ ├── keys │ │ │ │ ├── DerivedEncryptionKey.java │ │ │ │ ├── DerivedRootEncryptionKey.java │ │ │ │ ├── HmacKeyDerivationFunction.java │ │ │ │ ├── Key.java │ │ │ │ ├── KeyUtil.java │ │ │ │ ├── SaltedHkdf.java │ │ │ │ └── package-info.java │ │ │ ├── materials │ │ │ │ ├── AbstractRawMaterials.java │ │ │ │ ├── CryptographicMaterials.java │ │ │ │ ├── DecryptionMaterials.java │ │ │ │ ├── EncryptionMaterials.java │ │ │ │ ├── SymmetricRawMaterials.java │ │ │ │ └── package-info.java │ │ │ ├── package-info.java │ │ │ └── providers │ │ │ │ ├── EncryptionMaterialsProvider.java │ │ │ │ ├── SymmetricStaticProvider.java │ │ │ │ └── package-info.java │ │ │ ├── exception │ │ │ ├── C3rIllegalArgumentException.java │ │ │ ├── C3rRuntimeException.java │ │ │ └── package-info.java │ │ │ ├── internal │ │ │ ├── AdditionalAuthenticatedData.java │ │ │ ├── InitializationVector.java │ │ │ ├── Limits.java │ │ │ ├── Nonce.java │ │ │ ├── PadUtil.java │ │ │ ├── Validatable.java │ │ │ └── package-info.java │ │ │ ├── io │ │ │ ├── CsvRowReader.java │ │ │ ├── CsvRowWriter.java │ │ │ ├── FileFormat.java │ │ │ ├── RowReader.java │ │ │ ├── RowWriter.java │ │ │ ├── SqlRowReader.java │ │ │ ├── SqlRowWriter.java │ │ │ ├── package-info.java │ │ │ └── sql │ │ │ │ ├── SqlTable.java │ │ │ │ ├── TableGenerator.java │ │ │ │ └── package-info.java │ │ │ ├── json │ │ │ ├── ColumnHeaderTypeAdapter.java │ │ │ ├── ColumnTypeTypeAdapter.java │ │ │ ├── GsonUtil.java │ │ │ ├── PadTypeTypeAdapter.java │ │ │ ├── TableSchemaTypeAdapter.java │ │ │ ├── ValidationTypeAdapterFactory.java │ │ │ └── package-info.java │ │ │ ├── package-info.java │ │ │ └── utils │ │ │ ├── C3rSdkProperties.java │ │ │ ├── FileUtil.java │ │ │ └── package-info.java │ └── resources │ │ └── log4j2.xml │ └── test │ ├── java │ └── com │ │ └── amazonaws │ │ └── c3r │ │ ├── CleartextTransformerTest.java │ │ ├── FingerprintTransformerTest.java │ │ ├── SealedTransformerTest.java │ │ ├── TransformerTest.java │ │ ├── action │ │ ├── CsvRowMarshallerTest.java │ │ ├── CsvRowUnmarshallerTest.java │ │ ├── RowMarshalPreserveNullTest.java │ │ ├── RowMarshallerTest.java │ │ └── RowUnmarshallerTest.java │ │ ├── cleanrooms │ │ └── CleanRoomsDaoTest.java │ │ ├── config │ │ ├── ColumnHeaderTest.java │ │ ├── ColumnInsightTest.java │ │ ├── ColumnSchemaTest.java │ │ ├── ConfigTest.java │ │ ├── DecryptConfigTest.java │ │ ├── EncryptConfigTest.java │ │ ├── MappedTableSchemaTest.java │ │ ├── PadTest.java │ │ ├── PositionalTableSchemaTest.java │ │ ├── TableSchemaCommonTestInterface.java │ │ └── TableSchemaTest.java │ │ ├── data │ │ ├── ClientDataInfoTest.java │ │ ├── ClientDataTypeMetadataTest.java │ │ ├── ClientDataTypeTest.java │ │ ├── CsvValueFactoryTest.java │ │ ├── CsvValueTest.java │ │ ├── FingerprintEquivalenceClassTest.java │ │ ├── UnitsTest.java │ │ ├── ValueConverterTest.java │ │ └── ValueTest.java │ │ ├── encryption │ │ ├── EncryptionContextTest.java │ │ ├── EncryptorTest.java │ │ ├── config │ │ │ └── ClientSettingsTest.java │ │ ├── keys │ │ │ ├── DerivedEncryptionKeyTest.java │ │ │ ├── DerivedRootEncryptionKeyTest.java │ │ │ ├── HmacKeyDerivationFunctionTest.java │ │ │ ├── KeyTest.java │ │ │ ├── KeyUtilTest.java │ │ │ └── SaltedHkdfTest.java │ │ ├── materials │ │ │ └── SymmetricRawMaterialsTest.java │ │ └── providers │ │ │ └── SymmetricStaticProviderTest.java │ │ ├── exception │ │ ├── C3rIllegalArgumentExceptionTest.java │ │ └── C3rRuntimeExceptionTest.java │ │ ├── internal │ │ ├── ColumnTypeTest.java │ │ ├── InitializationVectorTest.java │ │ ├── NonceTest.java │ │ └── PadUtilTest.java │ │ ├── io │ │ ├── CsvRowReaderTest.java │ │ ├── CsvRowWriterTest.java │ │ ├── CsvTestUtility.java │ │ ├── FileFormatTest.java │ │ ├── RowReaderTest.java │ │ ├── RowReaderTestUtility.java │ │ ├── SqlRowReaderTest.java │ │ ├── SqlRowWriterTest.java │ │ └── sql │ │ │ └── TableGeneratorTest.java │ │ ├── json │ │ ├── ColumnHeaderTypeAdapterTest.java │ │ ├── ColumnTypeTypeAdapterTest.java │ │ ├── GsonUtilTest.java │ │ ├── MappedTableSchemaTypeAdapterTest.java │ │ ├── PadTypeTypeAdapterTest.java │ │ ├── PositionalTableSchemaTypeAdapterTest.java │ │ ├── TableSchemaCommonTypeAdapterTestInterface.java │ │ ├── TableSchemaTypeAdapterTest.java │ │ └── ValidationTypeAdapterFactoryTest.java │ │ └── utils │ │ ├── C3rSdkPropertiesTest.java │ │ ├── DecryptSdkConfigTestUtility.java │ │ ├── EncryptSdkConfigTestUtility.java │ │ ├── FileTestUtility.java │ │ ├── FileUtilTest.java │ │ └── GeneralTestUtility.java │ └── resources │ └── log4j2.xml ├── c3r-sdk-examples ├── build.gradle └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── amazonaws │ │ │ └── c3r │ │ │ └── examples │ │ │ ├── csv │ │ │ ├── CsvExample.java │ │ │ ├── CsvNoHeaderExample.java │ │ │ └── package-info.java │ │ │ ├── parquet │ │ │ ├── ParquetExample.java │ │ │ └── package-info.java │ │ │ └── spark │ │ │ ├── SparkExample.java │ │ │ └── package-info.java │ └── resources │ │ └── log4j2.xml │ └── test │ ├── java │ └── com │ │ └── amazonaws │ │ └── c3r │ │ └── examples │ │ ├── csv │ │ ├── CsvExampleTest.java │ │ └── CsvNoHeaderExampleTest.java │ │ ├── parquet │ │ └── ParquetExampleTest.java │ │ ├── spark │ │ └── SparkExampleTest.java │ │ └── utils │ │ └── FileTestUtility.java │ └── resources │ └── log4j2.xml ├── c3r-sdk-parquet ├── build.gradle └── src │ ├── main │ └── java │ │ └── com │ │ └── amazonaws │ │ └── c3r │ │ ├── action │ │ ├── ParquetRowMarshaller.java │ │ ├── ParquetRowUnmarshaller.java │ │ └── package-info.java │ │ ├── config │ │ ├── ParquetConfig.java │ │ └── package-info.java │ │ ├── data │ │ ├── ParquetDataType.java │ │ ├── ParquetRow.java │ │ ├── ParquetSchema.java │ │ ├── ParquetValue.java │ │ ├── ParquetValueFactory.java │ │ └── package-info.java │ │ └── io │ │ ├── ParquetRowReader.java │ │ ├── ParquetRowWriter.java │ │ ├── package-info.java │ │ └── parquet │ │ ├── ParquetPrimitiveConverter.java │ │ ├── ParquetRowConverter.java │ │ ├── ParquetRowMaterializer.java │ │ ├── ParquetRowWriteSupport.java │ │ ├── ParquetWriterBuilder.java │ │ └── package-info.java │ └── test │ └── java │ └── com │ └── amazonaws │ └── c3r │ ├── action │ ├── ParquetRowMarshallerTest.java │ └── ParquetRowUnmarshallerTest.java │ ├── data │ ├── BadParquetAnnotationsTest.java │ ├── ParquetDataTypeTest.java │ ├── ParquetEquivalenceTypesTest.java │ ├── ParquetSchemaTest.java │ ├── ParquetValueFactoryTest.java │ └── ParquetValueTest.java │ ├── io │ ├── ParquetRowReaderTest.java │ ├── ParquetRowWriterTest.java │ └── parquet │ │ └── ParquetPrimitiveConverterTest.java │ └── utils │ ├── FileTestUtility.java │ ├── ParquetTestUtility.java │ └── ParquetTypeDefsTestUtility.java ├── codebuild ├── build-and-test-release-artifacts.sh ├── release.yml ├── release │ ├── release-prod.yml │ ├── release-staging.yml │ ├── release.yml │ ├── settings.xml │ └── upload-artifacts.yml └── sign_artifacts.py ├── config ├── checkstyle │ ├── checkstyle.xml │ ├── license-header.txt │ └── suppressions.xml └── spotbugs │ └── excludeFilter.xml ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── lombok.config ├── samples ├── csv │ ├── customNull5by6.csv │ ├── data_sample_no_headers.csv │ ├── data_sample_with_quotes.csv │ ├── data_sample_without_quotes.csv │ ├── empty5by6.csv │ ├── marshalled_data_sample.csv │ ├── nonUtf8Encoding.csv │ ├── null5by6.csv │ ├── one_row_null_sample.csv │ ├── one_row_null_sample_special_null.csv │ ├── one_row_too_few_columns.csv │ └── one_row_too_many_columns.csv ├── parquet │ ├── all_valid_types.parquet │ ├── all_valid_types_and_list.parquet │ ├── binary_values.parquet │ ├── data_sample.parquet │ ├── data_sample_with_non_string_types.parquet │ ├── nonUtf8Encoding.parquet │ ├── null_rows_100_groups_1_prim_data.parquet │ ├── null_rows_1_groups_1_prim_data.parquet │ ├── rows_100_groups_10_prim_data.parquet │ ├── rows_100_groups_1_prim_data.parquet │ ├── rows_1_groups_1_prim_data.parquet │ └── supported_and_unsupported_types.parquet └── schema │ ├── 6column.json │ ├── all_valid_types_and_list_schema.json │ ├── config_sample.json │ ├── config_sample_no_cleartext.json │ ├── config_sample_no_headers.json │ ├── config_sample_x3.json │ └── supported_and_unsupported_types.json ├── settings.gradle └── version.txt /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Each line is a file pattern followed by one or more owners. 2 | # https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners 3 | 4 | # Default code owner for everything is our aws-crypto-tools group 5 | * @aws/aws-crypto-compute -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Issue Report 2 | 3 | (A clear and concise description of the issue) 4 | 5 | ## Expected Behavior 6 | 7 | (Write out the expected behavior here.) 8 | 9 | ## Actual Behavior 10 | 11 | (Write out what actually happened here.) 12 | 13 | ## Steps to Reproduce the Issue 14 | 15 | (Write out detailed steps to reproduce the issue here.) -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | *Issue #, if available:* 2 | 3 | *Description of changes:* 4 | 5 | 6 | By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "gradle" 9 | directory: "/" 10 | schedule: 11 | interval: "daily" 12 | time: "08:00" 13 | commit-message: 14 | prefix: "build(gradle)" 15 | ignore: 16 | # Temporary until conflicts with new Spotbugs version may be updated 17 | - dependency-name: "com.github.spotbugs" 18 | - package-ecosystem: "github-actions" 19 | directory: "/" 20 | schedule: 21 | interval: "daily" 22 | time: "08:00" 23 | commit-message: 24 | prefix: "build(CI)" 25 | -------------------------------------------------------------------------------- /.github/workflows/dependabot-auto-merge.yml: -------------------------------------------------------------------------------- 1 | name: Dependabot auto-merge 2 | on: pull_request 3 | 4 | permissions: 5 | contents: write 6 | pull-requests: write 7 | 8 | jobs: 9 | dependabot: 10 | runs-on: ubuntu-latest 11 | if: ${{ github.actor == 'dependabot[bot]' }} 12 | steps: 13 | - name: Approve a PR 14 | run: gh pr review --approve "$PR_URL" 15 | env: 16 | PR_URL: ${{github.event.pull_request.html_url}} 17 | GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} 18 | - name: Enable auto-merge for Dependabot PRs 19 | run: gh pr merge --auto --squash "$PR_URL" 20 | env: 21 | PR_URL: ${{github.event.pull_request.html_url}} 22 | GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} -------------------------------------------------------------------------------- /.github/workflows/jacoco.yml: -------------------------------------------------------------------------------- 1 | name: JaCoCo Report 2 | on: [ pull_request ] 3 | 4 | jobs: 5 | # Adds the following filters: 6 | # Only run when files that influence the build are updated. 7 | # Don't run again if the same commit has already been run. 8 | filter: 9 | runs-on: ubuntu-latest 10 | outputs: 11 | should_skip: ${{ steps.skip_check.outputs.should_skip }} 12 | steps: 13 | - id: skip_check 14 | uses: fkirc/skip-duplicate-actions@v5.3.1 15 | with: 16 | concurrent_skipping: 'same_content_newer' 17 | paths: '["**/src/**", "**.gradle", "samples/**", "config/**"]' 18 | 19 | generate-jacoco: 20 | strategy: 21 | matrix: 22 | package: [ 'c3r-sdk-core' , 'c3r-sdk-parquet' , 'c3r-cli' ] 23 | runs-on: ubuntu-latest 24 | needs: filter 25 | if: ${{ needs.filter.outputs.should_skip != 'true' }} 26 | steps: 27 | - name: Checkout code 28 | uses: actions/checkout@v4 29 | - name: Set up JDK 30 | uses: actions/setup-java@v4 31 | with: 32 | java-version: '11' 33 | distribution: 'corretto' 34 | - name: Set up Gradle 35 | uses: gradle/actions/setup-gradle@v4 36 | - name: Run Gradle build 37 | run: ./gradlew build --parallel 38 | - name: jacoco-badge-generator 39 | id: jacoco 40 | uses: cicirello/jacoco-badge-generator@v2 41 | with: 42 | generate-branches-badge: true 43 | jacoco-csv-file: ${{ matrix.package }}/build/reports/jacoco/test/jacocoTestReport.csv 44 | coverage-badge-filename: ${{ matrix.package }}_coverage.svg 45 | coverage-label: coverage ${{ matrix.package }} 46 | branches-badge-filename: ${{ matrix.package }}_branches.svg 47 | branches-label: branches ${{ matrix.package }} 48 | 49 | - name: Log coverage percentages to workflow output 50 | run: | 51 | echo "coverage = ${{ steps.jacoco.outputs.coverage }}" 52 | echo "branches = ${{ steps.jacoco.outputs.branches }}" 53 | -------------------------------------------------------------------------------- /.github/workflows/markdown-link-check.yml: -------------------------------------------------------------------------------- 1 | name: Check Markdown links 2 | 3 | # Run on push, PRs, and every Sunday at 9 am 4 | on: 5 | push: 6 | pull_request: 7 | schedule: 8 | - cron: "0 9 * * 0" 9 | 10 | jobs: 11 | markdown-link-check: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@master 15 | - uses: gaurav-nelson/github-action-markdown-link-check@v1 16 | with: 17 | use-quiet-mode: 'yes' 18 | use-verbose-mode: 'yes' 19 | -------------------------------------------------------------------------------- /.github/workflows/new_ticket.yml: -------------------------------------------------------------------------------- 1 | name: New Ticket 2 | on: 3 | issues: 4 | types: 5 | - reopened 6 | - opened 7 | jobs: 8 | label_issues: 9 | runs-on: ubuntu-latest 10 | permissions: 11 | issues: write 12 | steps: 13 | - uses: actions/github-script@v7 14 | with: 15 | script: | 16 | github.rest.issues.addLabels({ 17 | issue_number: context.issue.number, 18 | owner: context.repo.owner, 19 | repo: context.repo.repo, 20 | labels: ["needs-response"] 21 | }) 22 | -------------------------------------------------------------------------------- /.github/workflows/pr-lint.yml: -------------------------------------------------------------------------------- 1 | # Enforce PR title following the Conventional Commit standard. 2 | # See references for more details: 3 | # - https://www.conventionalcommits.org/ 4 | # - See https://github.com/amannn/action-semantic-pull-request 5 | name: "Lint PR" 6 | 7 | on: 8 | pull_request_target: 9 | types: 10 | - opened 11 | - edited 12 | - synchronize 13 | 14 | jobs: 15 | main: 16 | name: Validate PR title 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: amannn/action-semantic-pull-request@v5 20 | id: lint_pr_title 21 | env: 22 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 23 | - uses: marocchino/sticky-pull-request-comment@v2 24 | # When the previous steps fail, the workflow would stop. By adding this 25 | # condition you can continue the execution with the populated error message. 26 | if: always() && (steps.lint_pr_title.outputs.error_message != null) 27 | with: 28 | header: pr-title-lint-error 29 | message: | 30 | Hey there and thank you for opening this pull request! 31 | 32 | We require pull request titles to follow the [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/) and it looks like your proposed title needs to be adjusted. 33 | 34 | Details: 35 | 36 | ``` 37 | ${{ steps.lint_pr_title.outputs.error_message }} 38 | ``` 39 | 40 | # Delete a previous comment when the issue has been resolved 41 | - if: ${{ steps.lint_pr_title.outputs.error_message == null }} 42 | uses: marocchino/sticky-pull-request-comment@v2 43 | with: 44 | header: pr-title-lint-error 45 | delete: true -------------------------------------------------------------------------------- /.github/workflows/repo-sync.yml: -------------------------------------------------------------------------------- 1 | name: Repo Sync 2 | on: 3 | schedule: 4 | - cron: '0 0 * * *' # run at midnight every day 5 | workflow_dispatch: # enable manual runs from GitHub UI 6 | 7 | jobs: 8 | repo-sync: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v4 12 | - name: repo-sync 13 | uses: repo-sync/github-sync@v2 14 | with: 15 | source_repo: "https://github.com/aws/c3r.git" 16 | source_branch: "main" 17 | destination_branch: "main" 18 | github_token: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /.github/workflows/trufflehog.yml: -------------------------------------------------------------------------------- 1 | name: Leaked Secrets Scan 2 | on: [pull_request] 3 | jobs: 4 | TruffleHog: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - name: Checkout code 8 | uses: actions/checkout@v4 9 | with: 10 | fetch-depth: 0 11 | - name: TruffleHog OSS 12 | uses: trufflesecurity/trufflehog@v3.88.35 13 | with: 14 | path: ./ 15 | base: ${{ github.event.repository.default_branch }} 16 | head: HEAD 17 | extra_args: --debug --only-verified -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # IntelliJ files 2 | **/*.iml 3 | **/*.idea/* 4 | **/*.attach_pid* 5 | 6 | # Compiled class files 7 | 8 | **/*.class 9 | /classes 10 | /annotation-generated-src/ 11 | 12 | # Log files 13 | 14 | **/*.log 15 | logs/ 16 | 17 | # Gradle files 18 | 19 | .gradle 20 | **/*gradle.properties 21 | 22 | # Other temp files 23 | 24 | out/ 25 | **/*.out 26 | **/*.db 27 | **/*.tmp 28 | **/*.out.csv 29 | **/*.out.parquet 30 | **/*.out.crc 31 | **/*.out.unknown 32 | build/ 33 | bin/ -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Cryptographic Computing for Clean Rooms (C3R) 2 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | ## Reporting a Vulnerability 2 | 3 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security 4 | via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) or directly via email to aws-security@amazon.com. 5 | Please do **not** create a public GitHub issue. -------------------------------------------------------------------------------- /SUPPORT.md: -------------------------------------------------------------------------------- 1 | If you are having trouble running C3R, make sure you have reviewed the [AWS Clean Rooms User Guide](https://docs.aws.amazon.com/clean-rooms/latest/userguide/index.html) and the [Getting Started](https://github.com/aws/c3r#getting-started) section of the README. 2 | 3 | If you find a bug, please submit it to our issue tracker at https://github.com/aws/c3r/issues and label it as a bug. -------------------------------------------------------------------------------- /c3r-cli-spark/src/main/java/com/amazonaws/c3r/spark/action/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * This package classes used to marshall (encrypt) and unmarshall (decrypt) data to and from the clean room for the various supported 6 | * data types. {@link com.amazonaws.c3r.spark.action.SparkMarshaller} handles the logic of marshalling data outside of anything having to 7 | * do with the actual data format and {@link com.amazonaws.c3r.spark.action.SparkUnmarshaller} does the same for unmarshalling. Each format 8 | * specific class handles file I/O and value creation only for that particular data type. 9 | * 10 | *

11 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 12 | * SPDX-License-Identifier: Apache-2.0 13 | */ 14 | package com.amazonaws.c3r.spark.action; -------------------------------------------------------------------------------- /c3r-cli-spark/src/main/java/com/amazonaws/c3r/spark/cli/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Command line interface for using Cryptographic Computing for Clean Rooms with AWS Clean Rooms. 6 | * 7 | *

8 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 9 | * SPDX-License-Identifier: Apache-2.0 10 | */ 11 | @DefaultAnnotation(NonNull.class) 12 | package com.amazonaws.c3r.spark.cli; 13 | 14 | import edu.umd.cs.findbugs.annotations.DefaultAnnotation; 15 | import lombok.NonNull; -------------------------------------------------------------------------------- /c3r-cli-spark/src/main/java/com/amazonaws/c3r/spark/config/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Classes that contain all the information needed to perform cryptographic computations on input data. 6 | * 7 | *

8 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 9 | * SPDX-License-Identifier: Apache-2.0 10 | */ 11 | package com.amazonaws.c3r.spark.config; -------------------------------------------------------------------------------- /c3r-cli-spark/src/main/java/com/amazonaws/c3r/spark/io/csv/Csv.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.spark.io.csv; 5 | 6 | import org.apache.spark.sql.connector.catalog.Table; 7 | import org.apache.spark.sql.connector.catalog.TableProvider; 8 | import org.apache.spark.sql.connector.expressions.Transform; 9 | import org.apache.spark.sql.types.StructType; 10 | import org.apache.spark.sql.util.CaseInsensitiveStringMap; 11 | 12 | import java.util.Map; 13 | 14 | /** 15 | * Custom CSV DataSource for Spark. Using this custom DataSource in place of Spark's built-in functionality allows us to maintain tighter 16 | * controls of edge cases like {@code null}, quoted empty space, and custom null values. 17 | */ 18 | public class Csv implements TableProvider { 19 | 20 | /** 21 | * {@inheritDoc} 22 | */ 23 | @Override 24 | public StructType inferSchema(final CaseInsensitiveStringMap options) { 25 | return SchemaUtil.inferSchema(options); 26 | } 27 | 28 | /** 29 | * {@inheritDoc} 30 | */ 31 | @Override 32 | public Table getTable(final StructType schema, final Transform[] partitioning, final Map properties) { 33 | return new CsvTable(schema, properties); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /c3r-cli-spark/src/main/java/com/amazonaws/c3r/spark/io/csv/CsvBatch.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.spark.io.csv; 5 | 6 | import lombok.AllArgsConstructor; 7 | import org.apache.spark.sql.connector.read.Batch; 8 | import org.apache.spark.sql.connector.read.InputPartition; 9 | import org.apache.spark.sql.connector.read.PartitionReaderFactory; 10 | 11 | import java.util.Map; 12 | 13 | /** 14 | * A physical representation of a data source scan for batch queries. This interface is used to provide physical information, like how 15 | * many partitions the scanned data has, and how to read records from the partitions. 16 | */ 17 | @AllArgsConstructor 18 | public class CsvBatch implements Batch { 19 | 20 | /** 21 | * A map of configuration settings. 22 | */ 23 | private final Map properties; 24 | 25 | /** 26 | * {@inheritDoc} 27 | */ 28 | @Override 29 | public InputPartition[] planInputPartitions() { 30 | return new InputPartition[]{new CsvInputPartition()}; 31 | } 32 | 33 | /** 34 | * {@inheritDoc} 35 | */ 36 | @Override 37 | public PartitionReaderFactory createReaderFactory() { 38 | return new CsvPartitionReaderFactory(properties); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /c3r-cli-spark/src/main/java/com/amazonaws/c3r/spark/io/csv/CsvBatchWrite.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.spark.io.csv; 5 | 6 | import lombok.AllArgsConstructor; 7 | import org.apache.spark.sql.connector.write.BatchWrite; 8 | import org.apache.spark.sql.connector.write.DataWriterFactory; 9 | import org.apache.spark.sql.connector.write.PhysicalWriteInfo; 10 | import org.apache.spark.sql.connector.write.WriterCommitMessage; 11 | 12 | import java.util.Map; 13 | 14 | /** 15 | * An implementation of BatchWrite that defines how to write the data to data source for batch processing. 16 | */ 17 | @AllArgsConstructor 18 | public class CsvBatchWrite implements BatchWrite { 19 | 20 | /** 21 | * A map of configuration settings. 22 | */ 23 | private final Map properties; 24 | 25 | /** 26 | * {@inheritDoc} 27 | */ 28 | @Override 29 | public DataWriterFactory createBatchWriterFactory(final PhysicalWriteInfo info) { 30 | return new CsvDataWriterFactory(properties); 31 | } 32 | 33 | /** 34 | * {@inheritDoc} 35 | */ 36 | @Override 37 | public void commit(final WriterCommitMessage[] messages) { 38 | // no-op 39 | } 40 | 41 | /** 42 | * {@inheritDoc} 43 | */ 44 | @Override 45 | public void abort(final WriterCommitMessage[] messages) { 46 | // no-op 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /c3r-cli-spark/src/main/java/com/amazonaws/c3r/spark/io/csv/CsvDataWriterFactory.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.spark.io.csv; 5 | 6 | import lombok.AllArgsConstructor; 7 | import org.apache.spark.sql.catalyst.InternalRow; 8 | import org.apache.spark.sql.connector.write.DataWriter; 9 | import org.apache.spark.sql.connector.write.DataWriterFactory; 10 | 11 | import java.util.Map; 12 | 13 | /** 14 | * A factory of DataWriter returned by {@code BatchWrite.createBatchWriterFactory(PhysicalWriteInfo)}, which is responsible for creating and 15 | * initializing the actual data writer at executor side. 16 | * 17 | *

18 | * Note that, the writer factory will be serialized and sent to executors, then the data writer will be created on executors and do the 19 | * actual writing. So this interface must be serializable and DataWriter doesn't need to be. 20 | */ 21 | @AllArgsConstructor 22 | public class CsvDataWriterFactory implements DataWriterFactory { 23 | /** 24 | * A map of configuration settings. 25 | */ 26 | private final Map properties; 27 | 28 | /** 29 | * {@inheritDoc} 30 | */ 31 | @Override 32 | public DataWriter createWriter(final int partitionId, final long taskId) { 33 | return new CsvDataWriter(partitionId, properties); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /c3r-cli-spark/src/main/java/com/amazonaws/c3r/spark/io/csv/CsvInputPartition.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.spark.io.csv; 5 | 6 | import org.apache.spark.sql.connector.read.InputPartition; 7 | 8 | /** 9 | * A serializable representation of an input partition returned by {@code Batch.planInputPartitions()} and the corresponding ones in 10 | * streaming. 11 | * 12 | *

13 | * Note that InputPartition will be serialized and sent to executors, then PartitionReader will be created by {@code PartitionReaderFactory 14 | * .createReader(InputPartition)} or {@code PartitionReaderFactory.createColumnarReader(InputPartition)} on executors to do the actual 15 | * reading. So InputPartition must be serializable while PartitionReader doesn't need to be. 16 | */ 17 | public class CsvInputPartition implements InputPartition { 18 | 19 | } 20 | -------------------------------------------------------------------------------- /c3r-cli-spark/src/main/java/com/amazonaws/c3r/spark/io/csv/CsvPartitionReader.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.spark.io.csv; 5 | 6 | import com.amazonaws.c3r.config.ColumnHeader; 7 | import com.amazonaws.c3r.data.CsvValue; 8 | import com.amazonaws.c3r.data.Row; 9 | import com.amazonaws.c3r.io.CsvRowReader; 10 | import org.apache.spark.sql.catalyst.InternalRow; 11 | import org.apache.spark.sql.connector.read.PartitionReader; 12 | import org.apache.spark.unsafe.types.UTF8String; 13 | import scala.collection.JavaConverters; 14 | 15 | import java.util.Arrays; 16 | import java.util.List; 17 | import java.util.Map; 18 | 19 | /** 20 | * A partition reader returned by {@code PartitionReaderFactory.createReader(InputPartition)} or {@code PartitionReaderFactory 21 | * .createColumnarReader(InputPartition)}. It's responsible for outputting data for a RDD partition. 22 | */ 23 | public class CsvPartitionReader implements PartitionReader { 24 | 25 | /** 26 | * Reader for processing CSV files. 27 | */ 28 | private final CsvRowReader csvReader; 29 | 30 | /** 31 | * Constructs a new CsvPartitionReader. 32 | * 33 | * @param properties A map of configuration settings 34 | */ 35 | public CsvPartitionReader(final Map properties) { 36 | this.csvReader = SparkCsvReader.initReader(properties); 37 | } 38 | 39 | /** 40 | * {@inheritDoc} 41 | */ 42 | @Override 43 | public boolean next() { 44 | return csvReader.hasNext(); 45 | } 46 | 47 | /** 48 | * {@inheritDoc} 49 | */ 50 | @Override 51 | public InternalRow get() { 52 | final Row row = csvReader.next(); 53 | final List headers = csvReader.getHeaders(); 54 | final Object[] data = new Object[row.size()]; 55 | for (int i = 0; i < data.length; i++) { 56 | final CsvValue val = row.getValue(headers.get(i)); 57 | data[i] = val.isNull() ? null : UTF8String.fromString(val.toString()); 58 | } 59 | return InternalRow.apply(JavaConverters.asScalaIteratorConverter(Arrays.asList(data).iterator()).asScala().toSeq()); 60 | } 61 | 62 | /** 63 | * {@inheritDoc} 64 | */ 65 | @Override 66 | public void close() { 67 | csvReader.close(); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /c3r-cli-spark/src/main/java/com/amazonaws/c3r/spark/io/csv/CsvPartitionReaderFactory.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.spark.io.csv; 5 | 6 | import lombok.AllArgsConstructor; 7 | import org.apache.spark.sql.catalyst.InternalRow; 8 | import org.apache.spark.sql.connector.read.InputPartition; 9 | import org.apache.spark.sql.connector.read.PartitionReader; 10 | import org.apache.spark.sql.connector.read.PartitionReaderFactory; 11 | 12 | import java.util.Map; 13 | 14 | /** 15 | * A factory used to create PartitionReader instances. 16 | * 17 | *

18 | * If Spark fails to execute any methods in the implementations of this interface or in the returned PartitionReader (by throwing an 19 | * exception), corresponding Spark task would fail and get retried until hitting the maximum retry times. 20 | */ 21 | @AllArgsConstructor 22 | public class CsvPartitionReaderFactory implements PartitionReaderFactory { 23 | 24 | /** 25 | * A map of configuration settings. 26 | */ 27 | private final Map properties; 28 | 29 | /** 30 | * {@inheritDoc} 31 | */ 32 | @Override 33 | public PartitionReader createReader(final InputPartition partition) { 34 | return new CsvPartitionReader(properties); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /c3r-cli-spark/src/main/java/com/amazonaws/c3r/spark/io/csv/CsvScan.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.spark.io.csv; 5 | 6 | import lombok.AllArgsConstructor; 7 | import lombok.Getter; 8 | import lombok.experimental.Accessors; 9 | import org.apache.spark.sql.connector.read.Batch; 10 | import org.apache.spark.sql.connector.read.Scan; 11 | import org.apache.spark.sql.types.StructType; 12 | 13 | import java.util.Map; 14 | 15 | /** 16 | * A logical representation of a data source scan. This interface is used to provide logical information, like what the actual read 17 | * schema is. 18 | */ 19 | @AllArgsConstructor 20 | public class CsvScan implements Scan { 21 | 22 | /** 23 | * A schema representation of the CSV file. 24 | */ 25 | @Getter 26 | @Accessors(fluent = true) 27 | private final StructType readSchema; 28 | 29 | /** 30 | * A map of configuration settings. 31 | */ 32 | private final Map properties; 33 | 34 | /** 35 | * {@inheritDoc} 36 | */ 37 | @Override 38 | public Batch toBatch() { 39 | return new CsvBatch(properties); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /c3r-cli-spark/src/main/java/com/amazonaws/c3r/spark/io/csv/CsvScanBuilder.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.spark.io.csv; 5 | 6 | import lombok.AllArgsConstructor; 7 | import org.apache.spark.sql.connector.read.Scan; 8 | import org.apache.spark.sql.connector.read.ScanBuilder; 9 | import org.apache.spark.sql.types.StructType; 10 | 11 | import java.util.Map; 12 | 13 | /** 14 | * An implementation of ScanBuilder for building the Scan. 15 | */ 16 | @AllArgsConstructor 17 | public class CsvScanBuilder implements ScanBuilder { 18 | 19 | /** 20 | * A schema representation of the CSV file. 21 | */ 22 | private final StructType schema; 23 | 24 | /** 25 | * A map of configuration settings. 26 | */ 27 | private final Map properties; 28 | 29 | /** 30 | * {@inheritDoc} 31 | */ 32 | @Override 33 | public Scan build() { 34 | return new CsvScan(schema, properties); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /c3r-cli-spark/src/main/java/com/amazonaws/c3r/spark/io/csv/CsvWrite.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.spark.io.csv; 5 | 6 | import lombok.AllArgsConstructor; 7 | import org.apache.spark.sql.connector.write.BatchWrite; 8 | import org.apache.spark.sql.connector.write.Write; 9 | 10 | import java.util.Map; 11 | 12 | /** 13 | * A logical representation of a data source write. 14 | */ 15 | @AllArgsConstructor 16 | public class CsvWrite implements Write { 17 | 18 | /** 19 | * A map of configuration settings. 20 | */ 21 | private final Map properties; 22 | 23 | /** 24 | * {@inheritDoc} 25 | */ 26 | @Override 27 | public BatchWrite toBatch() { 28 | return new CsvBatchWrite(properties); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /c3r-cli-spark/src/main/java/com/amazonaws/c3r/spark/io/csv/CsvWriteBuilder.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.spark.io.csv; 5 | 6 | import lombok.AllArgsConstructor; 7 | import org.apache.spark.sql.connector.write.Write; 8 | import org.apache.spark.sql.connector.write.WriteBuilder; 9 | 10 | import java.util.Map; 11 | 12 | /** 13 | * An implementation of WriterBuilder for building the Write. 14 | */ 15 | @AllArgsConstructor 16 | public class CsvWriteBuilder implements WriteBuilder { 17 | 18 | /** 19 | * A map of configuration settings. 20 | */ 21 | private final Map properties; 22 | 23 | /** 24 | * {@inheritDoc} 25 | */ 26 | @Override 27 | public Write build() { 28 | return new CsvWrite(properties); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /c3r-cli-spark/src/main/java/com/amazonaws/c3r/spark/io/csv/SchemaUtil.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.spark.io.csv; 5 | 6 | import com.amazonaws.c3r.config.ColumnHeader; 7 | import com.amazonaws.c3r.io.CsvRowReader; 8 | import org.apache.spark.sql.types.DataTypes; 9 | import org.apache.spark.sql.types.StructField; 10 | import org.apache.spark.sql.types.StructType; 11 | 12 | import java.util.Arrays; 13 | import java.util.List; 14 | import java.util.Map; 15 | import java.util.stream.Collectors; 16 | 17 | /** 18 | * Utility class for creating schemas from ColumnHeaders. 19 | */ 20 | public abstract class SchemaUtil { 21 | 22 | /** 23 | * Create a schema based on the headers provided. 24 | * 25 | * @param headers The headers used to create a schema 26 | * @return a schema for the headers provided 27 | */ 28 | public static StructType inferSchema(final List headers) { 29 | final StructField[] fields = headers.stream().map(ColumnHeader::toString) 30 | .map(header -> DataTypes.createStructField(header, DataTypes.StringType, true)) 31 | .toArray(StructField[]::new); 32 | return DataTypes.createStructType(fields); 33 | } 34 | 35 | /** 36 | * Create a schema based on the headers provided. If no headers were provided, attempt to read the headers. 37 | * 38 | * @param properties A map of configuration settings 39 | * @return a schema for the headers provided 40 | */ 41 | public static StructType inferSchema(final Map properties) { 42 | if (!properties.containsKey("headers")) { 43 | final CsvRowReader reader = SparkCsvReader.initReader(properties); 44 | return SchemaUtil.inferSchema(reader.getHeaders()); 45 | } 46 | final String[] headers = properties.get("headers").split(","); 47 | final List fields = Arrays.stream(headers) 48 | .map(field -> DataTypes.createStructField(field, DataTypes.StringType, true)) 49 | .collect(Collectors.toList()); 50 | return DataTypes.createStructType(fields); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /c3r-cli-spark/src/main/java/com/amazonaws/c3r/spark/io/csv/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Support for reading and writing rows of CSV data. 6 | * 7 | *

8 | * Only the SparkCsvReader and SparkCsvWriter should be used for development. All other classes are subject to change without notice. 9 | * 10 | *

11 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 12 | * SPDX-License-Identifier: Apache-2.0 13 | */ 14 | package com.amazonaws.c3r.spark.io.csv; -------------------------------------------------------------------------------- /c3r-cli-spark/src/main/java/com/amazonaws/c3r/spark/io/parquet/SparkParquetWriter.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.spark.io.parquet; 5 | 6 | import lombok.NonNull; 7 | import org.apache.spark.sql.Dataset; 8 | import org.apache.spark.sql.Row; 9 | import org.apache.spark.sql.SaveMode; 10 | 11 | /** 12 | * Utility class for Spark to write Parquet files to disk. 13 | */ 14 | public abstract class SparkParquetWriter { 15 | 16 | /** 17 | * Writes the Dataset to the root path. 18 | * 19 | * @param dataset The data to write 20 | * @param targetName The target path to write to 21 | */ 22 | public static void writeOutput(@NonNull final Dataset dataset, 23 | @NonNull final String targetName) { 24 | dataset.write().mode(SaveMode.Append) 25 | .parquet(targetName); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /c3r-cli-spark/src/main/java/com/amazonaws/c3r/spark/io/parquet/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Support for reading and writing rows of Parquet data. 6 | * 7 | *

8 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 9 | * SPDX-License-Identifier: Apache-2.0 10 | */ 11 | package com.amazonaws.c3r.spark.io.parquet; -------------------------------------------------------------------------------- /c3r-cli-spark/src/main/java/com/amazonaws/c3r/spark/io/schema/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Utilities to create either a simple schema for how input data will be mapped to output data and a helper program for users 6 | * who want to be walked through creating a complete schema for their data. 7 | * 8 | *

9 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 10 | * SPDX-License-Identifier: Apache-2.0 11 | */ 12 | package com.amazonaws.c3r.spark.io.schema; -------------------------------------------------------------------------------- /c3r-cli-spark/src/main/java/com/amazonaws/c3r/spark/utils/C3rCliSparkProperties.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.spark.utils; 5 | 6 | import com.amazonaws.c3r.utils.C3rSdkProperties; 7 | import software.amazon.awssdk.core.ApiName; 8 | 9 | /** 10 | * C3R CLI for Apache Spark properties. 11 | */ 12 | public final class C3rCliSparkProperties { 13 | 14 | /** 15 | * Application name of C3R CLI client for Apache Spark. 16 | */ 17 | public static final String APP_NAME = "c3r-cli-spark"; 18 | 19 | /** 20 | * User agent for the C3R CLI. 21 | */ 22 | public static final ApiName API_NAME = ApiName.builder() 23 | .name(APP_NAME) 24 | .version(C3rSdkProperties.VERSION) 25 | .build(); 26 | 27 | /** 28 | * Hidden utility class constructor. 29 | */ 30 | private C3rCliSparkProperties() { 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /c3r-cli-spark/src/main/java/com/amazonaws/c3r/spark/utils/SparkSessionUtil.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.spark.utils; 5 | 6 | import com.amazonaws.c3r.exception.C3rRuntimeException; 7 | import org.apache.spark.sql.SparkSession; 8 | 9 | /** 10 | * Utility class for Spark Session functionality. 11 | */ 12 | public abstract class SparkSessionUtil { 13 | 14 | /** 15 | * Initializes a SparkSession object with the passed Spark Drive URL. 16 | * 17 | * @return A SparkSession connected to the Spark Driver 18 | * @throws C3rRuntimeException if the Spark Driver cannot be connected to 19 | */ 20 | public static SparkSession initSparkSession() { 21 | try { 22 | return SparkSession 23 | .builder() 24 | .appName("C3R") 25 | .getOrCreate(); 26 | } catch (Exception e) { 27 | throw new C3rRuntimeException("Could not connect to Spark server.", e); 28 | } 29 | } 30 | 31 | /** 32 | * Shut down the Spark session. 33 | * 34 | * @param spark the SparkSession to close 35 | */ 36 | public static void closeSparkSession(final SparkSession spark) { 37 | spark.stop(); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /c3r-cli-spark/src/main/java/com/amazonaws/c3r/spark/utils/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Utility classes that contain commonly used functionality across components. 6 | * 7 | *

8 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 9 | * SPDX-License-Identifier: Apache-2.0 10 | */ 11 | package com.amazonaws.c3r.spark.utils; -------------------------------------------------------------------------------- /c3r-cli-spark/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | logs 5 | 6 | 7 | 9 | 10 | %d{dd MMM yyyy HH:mm:ss,SSS} [%p] (%t) %c: %m%n%ex 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | [%p]: %m%n%ex 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /c3r-cli-spark/src/test/java/com/amazonaws/c3r/spark/cleanrooms/CleanRoomsDaoTestUtility.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.spark.cleanrooms; 5 | 6 | import com.amazonaws.c3r.cleanrooms.CleanRoomsDao; 7 | import org.mockito.stubbing.Answer; 8 | 9 | import static org.mockito.ArgumentMatchers.any; 10 | import static org.mockito.Mockito.when; 11 | 12 | public final class CleanRoomsDaoTestUtility { 13 | 14 | /** 15 | * Hidden utility class constructor. 16 | */ 17 | private CleanRoomsDaoTestUtility() { 18 | } 19 | 20 | public static CleanRoomsDao generateMockDao() { 21 | final CleanRoomsDao mockCleanRoomsDao = org.mockito.Mockito.mock(CleanRoomsDao.class); 22 | when(mockCleanRoomsDao.withProfile(any())).thenAnswer((Answer) (invocation) -> { 23 | when(mockCleanRoomsDao.getProfile()).thenReturn(invocation.getArgument(0)); 24 | return mockCleanRoomsDao; 25 | }); 26 | when(mockCleanRoomsDao.withRegion(any())).thenAnswer((Answer) (invocation) -> { 27 | when(mockCleanRoomsDao.getRegion()).thenReturn(invocation.getArgument(0)); 28 | return mockCleanRoomsDao; 29 | }); 30 | when(mockCleanRoomsDao.getRegion()).thenCallRealMethod(); 31 | return mockCleanRoomsDao; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /c3r-cli-spark/src/test/java/com/amazonaws/c3r/spark/cli/CliTestUtility.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.spark.cli; 5 | 6 | import com.amazonaws.c3r.cleanrooms.CleanRoomsDao; 7 | import com.amazonaws.c3r.spark.cleanrooms.CleanRoomsDaoTestUtility; 8 | import com.amazonaws.c3r.spark.utils.SparkSessionTestUtility; 9 | 10 | import static org.mockito.ArgumentMatchers.any; 11 | import static org.mockito.Mockito.when; 12 | 13 | /** 14 | * Utilities to interface with the CLI interface as if you were calling from the command line. 15 | */ 16 | public final class CliTestUtility { 17 | /** 18 | * Hidden utility class constructor. 19 | */ 20 | private CliTestUtility() { 21 | } 22 | 23 | /** 24 | * Runs the cli with a mock to replace an actual connection to AWS Clean Rooms. 25 | * 26 | * @param args Command line parameters for encrypt mode 27 | * @return {@value Main#SUCCESS} if no errors are encountered or {@value Main#FAILURE} 28 | */ 29 | public static int runWithoutCleanRooms(final EncryptCliConfigTestUtility args) { 30 | final CleanRoomsDao cleanRoomsDao; 31 | cleanRoomsDao = CleanRoomsDaoTestUtility.generateMockDao(); 32 | when(cleanRoomsDao.getCollaborationDataEncryptionMetadata(any())).thenReturn(args.getClientSettings()); 33 | return EncryptMode.getApp(cleanRoomsDao, SparkSessionTestUtility.initSparkSession()) 34 | .execute(args.toArrayWithoutMode()); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /c3r-cli-spark/src/test/java/com/amazonaws/c3r/spark/cli/MainEnvVarKeyInvalidTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.spark.cli; 5 | 6 | import com.amazonaws.c3r.spark.utils.SparkSessionTestUtility; 7 | import org.apache.spark.sql.SparkSession; 8 | import org.junit.jupiter.api.BeforeEach; 9 | import org.junit.jupiter.api.Test; 10 | import picocli.CommandLine; 11 | 12 | import static org.junit.jupiter.api.Assertions.assertNotEquals; 13 | 14 | /* 15 | * Tests specifically needing an invalid key in the environment 16 | * variable for the shared secret key. 17 | */ 18 | public class MainEnvVarKeyInvalidTest { 19 | private static final String ENC_INPUT_PATH = "../samples/csv/data_sample_without_quotes.csv"; 20 | 21 | private static final String SCHEMA_PATH = "../samples/schema/config_sample.json"; 22 | 23 | private static final String DEC_INPUT_PATH = "../samples/csv/marshalled_data_sample.csv"; 24 | 25 | private DecryptCliConfigTestUtility decArgs; 26 | 27 | private CommandLine decMain; 28 | 29 | private EncryptCliConfigTestUtility encArgs; 30 | 31 | private CommandLine encMain; 32 | 33 | public int runEncryptMainWithCliArgs() { 34 | return encMain.execute(encArgs.toArrayWithoutMode()); 35 | } 36 | 37 | public int runDecryptMainWithCliArgs() { 38 | return decMain.execute(decArgs.toArrayWithoutMode()); 39 | } 40 | 41 | @BeforeEach 42 | public void setup() { 43 | final SparkSession sparkSession = SparkSessionTestUtility.initSparkSession(); 44 | encArgs = EncryptCliConfigTestUtility.defaultDryRunTestArgs(ENC_INPUT_PATH, SCHEMA_PATH); 45 | encMain = EncryptMode.getApp(null, sparkSession); 46 | decArgs = DecryptCliConfigTestUtility.defaultDryRunTestArgs(DEC_INPUT_PATH); 47 | decMain = DecryptMode.getApp(sparkSession); 48 | } 49 | 50 | @Test 51 | public void validateEncryptSecretKeyInvalidTest() { 52 | assertNotEquals(0, runEncryptMainWithCliArgs()); 53 | } 54 | 55 | @Test 56 | public void validateDecryptSecretKeyInvalidTest() { 57 | assertNotEquals(0, runDecryptMainWithCliArgs()); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /c3r-cli-spark/src/test/java/com/amazonaws/c3r/spark/io/schema/CsvSchemaGeneratorTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.spark.io.schema; 5 | 6 | import com.amazonaws.c3r.exception.C3rRuntimeException; 7 | import com.amazonaws.c3r.spark.utils.FileTestUtility; 8 | import com.amazonaws.c3r.utils.FileUtil; 9 | import org.junit.jupiter.api.Test; 10 | 11 | import java.io.IOException; 12 | import java.nio.file.Path; 13 | 14 | import static com.amazonaws.c3r.spark.utils.GeneralTestUtility.DATA_SAMPLE_HEADERS; 15 | import static org.junit.jupiter.api.Assertions.assertEquals; 16 | import static org.junit.jupiter.api.Assertions.assertThrows; 17 | 18 | public class CsvSchemaGeneratorTest { 19 | 20 | private CsvSchemaGenerator getTestSchemaGenerator(final String file) throws IOException { 21 | final String output = FileTestUtility.resolve("schema.json").toString(); 22 | return CsvSchemaGenerator.builder() 23 | .inputCsvFile(file) 24 | .hasHeaders(true) 25 | .targetJsonFile(output) 26 | .overwrite(true) 27 | .build(); 28 | } 29 | 30 | @Test 31 | public void getSourceHeadersTest() throws IOException { 32 | assertEquals( 33 | DATA_SAMPLE_HEADERS, 34 | getTestSchemaGenerator(FileUtil.CURRENT_DIR + "/../samples/csv/data_sample_without_quotes.csv").getSourceHeaders()); 35 | } 36 | 37 | @Test 38 | public void getSourceColumnCountTest() throws IOException { 39 | assertEquals( 40 | DATA_SAMPLE_HEADERS.size(), 41 | getTestSchemaGenerator(FileUtil.CURRENT_DIR + "/../samples/csv/data_sample_without_quotes.csv").getSourceColumnCount()); 42 | } 43 | 44 | @Test 45 | public void emptyFileTest() throws IOException { 46 | final Path emptyCsvFile = FileTestUtility.createTempFile("empty", ".csv"); 47 | assertThrows(C3rRuntimeException.class, () -> 48 | getTestSchemaGenerator(emptyCsvFile.toString())); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /c3r-cli-spark/src/test/java/com/amazonaws/c3r/spark/utils/DecryptSdkConfigTestUtility.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.spark.utils; 5 | 6 | import lombok.Builder; 7 | import lombok.Getter; 8 | 9 | import javax.crypto.spec.SecretKeySpec; 10 | 11 | /** 12 | * Basic Decryption settings. 13 | */ 14 | @Builder 15 | @Getter 16 | public class DecryptSdkConfigTestUtility { 17 | /** 18 | * Key to use for decryption. 19 | */ 20 | @Builder.Default 21 | private SecretKeySpec key = null; 22 | 23 | /** 24 | * Salt for key generation. 25 | */ 26 | @Builder.Default 27 | private String salt = null; 28 | 29 | /** 30 | * Input file. 31 | */ 32 | @Builder.Default 33 | private String input = null; 34 | 35 | /** 36 | * Column header names. 37 | */ 38 | @Builder.Default 39 | private String[] columnHeaders = null; 40 | } 41 | -------------------------------------------------------------------------------- /c3r-cli-spark/src/test/java/com/amazonaws/c3r/spark/utils/EncryptSdkConfigTestUtility.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.spark.utils; 5 | 6 | import com.amazonaws.c3r.config.ClientSettings; 7 | import com.amazonaws.c3r.config.TableSchema; 8 | import lombok.Builder; 9 | import lombok.Getter; 10 | 11 | import javax.crypto.spec.SecretKeySpec; 12 | import java.util.List; 13 | 14 | /** 15 | * Basic configuration settings for encryption. 16 | */ 17 | @Builder 18 | @Getter 19 | public class EncryptSdkConfigTestUtility { 20 | /** 21 | * Schema specification. 22 | */ 23 | @Builder.Default 24 | private TableSchema schema = null; 25 | 26 | /** 27 | * Key to use for encryption. 28 | */ 29 | @Builder.Default 30 | private SecretKeySpec key = null; 31 | 32 | /** 33 | * Salt to use for key generation. 34 | */ 35 | @Builder.Default 36 | private String salt = null; 37 | 38 | /** 39 | * Security related parameters. 40 | */ 41 | @Builder.Default 42 | private ClientSettings settings = ClientSettings.lowAssuranceMode(); 43 | 44 | /** 45 | * Input file. 46 | */ 47 | @Builder.Default 48 | private String input = null; 49 | 50 | /** 51 | * Column headers in the input file. 52 | */ 53 | @Builder.Default 54 | private List inputColumnHeaders = null; 55 | 56 | /** 57 | * Column headers to use in the output file. 58 | */ 59 | @Builder.Default 60 | private List outputColumnHeaders = null; 61 | } 62 | -------------------------------------------------------------------------------- /c3r-cli-spark/src/test/java/com/amazonaws/c3r/spark/utils/SparkSessionTestUtility.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.spark.utils; 5 | 6 | import org.apache.spark.SparkConf; 7 | import org.apache.spark.sql.SparkSession; 8 | 9 | public abstract class SparkSessionTestUtility { 10 | 11 | /** 12 | * Initializes a SparkSession object with the passed Spark Drive URL. 13 | * 14 | * @return A SparkSession connected to the Spark Driver 15 | */ 16 | public static SparkSession initSparkSession() { 17 | // CHECKSTYLE:OFF 18 | final SparkConf conf = new SparkConf() 19 | .setAppName("C3R") 20 | .setMaster("local[*]"); 21 | // CHECKSTYLE:ON 22 | 23 | return SparkSession 24 | .builder() 25 | .config(conf) 26 | .getOrCreate(); 27 | } 28 | 29 | /** 30 | * Shut down the Spark session. 31 | * 32 | * @param spark the SparkSession to close 33 | */ 34 | public static void closeSparkSession(final SparkSession spark) { 35 | spark.stop(); 36 | } 37 | } -------------------------------------------------------------------------------- /c3r-cli-spark/src/test/java/com/amazonaws/c3r/spark/utils/StringTestUtility.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.spark.utils; 5 | 6 | import java.util.regex.Pattern; 7 | 8 | public final class StringTestUtility { 9 | 10 | private StringTestUtility() { 11 | } 12 | 13 | /** 14 | * Counts how many times a search string occurs (non-overlapping) in given string content. 15 | * 16 | * @param searchString String to search for 17 | * @param content Content to search in 18 | * @return The number of occurrences of the search string in the content. 19 | */ 20 | public static int countMatches(final String searchString, final String content) { 21 | return content.split(Pattern.quote(searchString), -1).length - 1; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /c3r-cli-spark/src/test/java/com/amazonaws/c3r/spark/utils/StringTestUtilityTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.spark.utils; 5 | 6 | import org.junit.jupiter.api.Test; 7 | 8 | import static org.junit.jupiter.api.Assertions.assertEquals; 9 | 10 | public class StringTestUtilityTest { 11 | 12 | @Test 13 | public void countMatchesTest() { 14 | assertEquals(0, StringTestUtility.countMatches("a", "")); 15 | assertEquals(0, StringTestUtility.countMatches("a", "b")); 16 | assertEquals(1, StringTestUtility.countMatches("a", "a")); 17 | assertEquals(1, StringTestUtility.countMatches("a", "abcd")); 18 | assertEquals(3, StringTestUtility.countMatches("a", "abcdabcdabcd")); 19 | assertEquals(3, StringTestUtility.countMatches("aa", "aaabcdaaabcdaaabcd")); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /c3r-cli-spark/src/test/java/com/amazonaws/c3r/spark/utils/TimingResultTestUtility.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.spark.utils; 5 | 6 | import com.amazonaws.c3r.config.ColumnType; 7 | import lombok.Builder; 8 | 9 | /** 10 | * Used to store performance testing metrics. 11 | */ 12 | @Builder 13 | public class TimingResultTestUtility { 14 | /** 15 | * Header names for timing results. 16 | */ 17 | public static final String[] HEADERS = { 18 | "Columns", 19 | "Rows", 20 | "Marshal Time (s)", 21 | "Unmarshal Time (s)", 22 | "Input Size (MB)", 23 | "Marshalled Size (MB)", 24 | "Unmarshalled Size (MB)", 25 | "Cleartext Columns", 26 | "Sealed Columns", 27 | "Fingerprint Columns", 28 | "Chars/Entry" 29 | }; 30 | 31 | /** 32 | * How many column types we are supporting. 33 | */ 34 | private static final int NUM_COL_TYPES = ColumnType.values().length; 35 | 36 | /** 37 | * Conversion factor for bytes to megabytes. 38 | */ 39 | private static final double MB = Math.pow(2, 20); 40 | 41 | /** 42 | * How many characters per entry in the input file. 43 | */ 44 | private Integer charsPerEntry; 45 | 46 | /** 47 | * Number of columns in the files. 48 | */ 49 | private Integer columnCount; 50 | 51 | /** 52 | * Number of rows in the files. 53 | */ 54 | private Long rowCount; 55 | 56 | /** 57 | * Size of original input file. 58 | */ 59 | private Long inputSizeBytes; 60 | 61 | /** 62 | * Time spent marshalling data. 63 | */ 64 | private Long marshalTimeSec; 65 | 66 | /** 67 | * Size of marshalled file. 68 | */ 69 | private Long marshalledSizeBytes; 70 | 71 | /** 72 | * Time spent unmarshalling data. 73 | */ 74 | private Long unmarshalTimeSec; 75 | 76 | /** 77 | * Size of the unmarshalled file. 78 | */ 79 | private Long unmarshalledSizeBytes; 80 | 81 | } -------------------------------------------------------------------------------- /c3r-cli-spark/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | logs 5 | 6 | 7 | 9 | 10 | %d{dd MMM yyyy HH:mm:ss,SSS} [%p] (%t) %c: %m%n%ex 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | %d{dd MMM yyyy HH:mm:ss,SSS} [%p] (%t) %c: %m%n%ex 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /c3r-cli/src/main/java/com/amazonaws/c3r/cli/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Command line interface for using Cryptographic Computing for Clean Rooms with AWS Clean Rooms. 6 | * 7 | *

8 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 9 | * SPDX-License-Identifier: Apache-2.0 10 | */ 11 | @DefaultAnnotation(NonNull.class) 12 | package com.amazonaws.c3r.cli; 13 | 14 | import edu.umd.cs.findbugs.annotations.DefaultAnnotation; 15 | import lombok.NonNull; -------------------------------------------------------------------------------- /c3r-cli/src/main/java/com/amazonaws/c3r/io/schema/ParquetSchemaGenerator.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.io.schema; 5 | 6 | import com.amazonaws.c3r.config.ClientSettings; 7 | import com.amazonaws.c3r.io.ParquetRowReader; 8 | import com.amazonaws.c3r.utils.FileUtil; 9 | import lombok.Builder; 10 | import lombok.NonNull; 11 | 12 | /** 13 | * Used to generate a schema file for a specific Parquet file. User can ask for either a simple, autogenerated schema or be walked through 14 | * the entire schema creation process. 15 | */ 16 | public final class ParquetSchemaGenerator extends SchemaGenerator { 17 | /** 18 | * Set up for schema generation and validate settings. 19 | * 20 | * @param inputParquetFile Parquet file to read header information from 21 | * @param targetJsonFile Where to save the schema 22 | * @param overwrite Whether the {@code targetJsonFile} should be overwritten (if it exists) 23 | * @param clientSettings Collaboration's client settings if provided, else {@code null} 24 | * @param binaryAsString If {@code true}, treat unannounced binary values as strings 25 | */ 26 | @Builder 27 | private ParquetSchemaGenerator(@NonNull final String inputParquetFile, 28 | @NonNull final String targetJsonFile, 29 | @NonNull final Boolean overwrite, 30 | final ClientSettings clientSettings, 31 | final Boolean binaryAsString) { 32 | super(inputParquetFile, targetJsonFile, overwrite, clientSettings); 33 | FileUtil.verifyReadableFile(inputParquetFile); 34 | final var reader = ParquetRowReader.builder().sourceName(inputParquetFile).binaryAsString(binaryAsString).build(); 35 | sourceHeaders = reader.getHeaders(); 36 | sourceColumnTypes = reader.getParquetSchema().getColumnClientDataTypes(); 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /c3r-cli/src/main/java/com/amazonaws/c3r/io/schema/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Utilities to create either a simple schema for how input data will be mapped to output data and a helper program for users 6 | * who want to be walked through creating a complete schema for their data. 7 | * 8 | *

9 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 10 | * SPDX-License-Identifier: Apache-2.0 11 | */ 12 | package com.amazonaws.c3r.io.schema; -------------------------------------------------------------------------------- /c3r-cli/src/main/java/com/amazonaws/c3r/utils/C3rCliProperties.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.utils; 5 | 6 | import com.amazonaws.c3r.cli.CliDescriptions; 7 | import software.amazon.awssdk.core.ApiName; 8 | 9 | /** 10 | * C3R CLI properties. 11 | */ 12 | public final class C3rCliProperties { 13 | /** 14 | * User agent for the C3R CLI. 15 | */ 16 | public static final ApiName API_NAME = ApiName.builder() 17 | .name(CliDescriptions.APP_NAME) 18 | .version(C3rSdkProperties.VERSION) 19 | .build(); 20 | 21 | /** 22 | * Hidden utility class constructor. 23 | */ 24 | private C3rCliProperties() { 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /c3r-cli/src/main/java/com/amazonaws/c3r/utils/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Utility classes that contain commonly used functionality across components. 6 | * 7 | *

8 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 9 | * SPDX-License-Identifier: Apache-2.0 10 | */ 11 | package com.amazonaws.c3r.utils; -------------------------------------------------------------------------------- /c3r-cli/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | logs 5 | 6 | 7 | 9 | 10 | %d{dd MMM yyyy HH:mm:ss,SSS} [%p] (%t) %c: %m%n%ex 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | [%p]: %m%n%ex 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /c3r-cli/src/test/java/com/amazonaws/c3r/cleanrooms/CleanRoomsDaoTestUtility.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.cleanrooms; 5 | 6 | import org.mockito.stubbing.Answer; 7 | 8 | import static org.mockito.ArgumentMatchers.any; 9 | import static org.mockito.Mockito.when; 10 | 11 | public final class CleanRoomsDaoTestUtility { 12 | 13 | /** 14 | * Hidden utility class constructor. 15 | */ 16 | private CleanRoomsDaoTestUtility() { 17 | } 18 | 19 | public static CleanRoomsDao generateMockDao() { 20 | final CleanRoomsDao mockCleanRoomsDao = org.mockito.Mockito.mock(CleanRoomsDao.class); 21 | when(mockCleanRoomsDao.withProfile(any())).thenAnswer((Answer) (invocation) -> { 22 | when(mockCleanRoomsDao.getProfile()).thenReturn(invocation.getArgument(0)); 23 | return mockCleanRoomsDao; 24 | }); 25 | when(mockCleanRoomsDao.withRegion(any())).thenAnswer((Answer) (invocation) -> { 26 | when(mockCleanRoomsDao.getRegion()).thenReturn(invocation.getArgument(0)); 27 | return mockCleanRoomsDao; 28 | }); 29 | when(mockCleanRoomsDao.getRegion()).thenCallRealMethod(); 30 | return mockCleanRoomsDao; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /c3r-cli/src/test/java/com/amazonaws/c3r/cli/CliTestUtility.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.cli; 5 | 6 | import com.amazonaws.c3r.cleanrooms.CleanRoomsDao; 7 | import com.amazonaws.c3r.cleanrooms.CleanRoomsDaoTestUtility; 8 | import picocli.CommandLine; 9 | 10 | import static org.mockito.ArgumentMatchers.any; 11 | import static org.mockito.Mockito.when; 12 | 13 | /** 14 | * Utilities to interface with the CLI interface as if you were calling from the command line. 15 | */ 16 | public final class CliTestUtility { 17 | /** 18 | * Hidden utility class constructor. 19 | */ 20 | private CliTestUtility() { 21 | } 22 | 23 | /** 24 | * Function to test cli options without loading the entire system. First argument must be one of the sub-modes: 25 | * encrypt, decrypt, schema. From there, the rest of the parameters should match what is required of that particular sub-mode. If extra 26 | * parameters are present, an error will be thrown or if required parameters are missing and error will be thrown. 27 | * 28 | * @param args Set of strings corresponding to a run of the software 29 | * @return A data structure that stores all stages of parsing from initial reading of parameters until final matches are made 30 | */ 31 | public static CommandLine.ParseResult verifyCliOptions(final String[] args) { 32 | return Main.getApp().parseArgs(args); 33 | } 34 | 35 | /** 36 | * Runs the cli with a mock to replace an actual connection to AWS Clean Rooms. 37 | * 38 | * @param args Command line parameters for encrypt mode 39 | * @return {@value Main#SUCCESS} if no errors are encountered or {@value Main#FAILURE} 40 | */ 41 | public static int runWithoutCleanRooms(final EncryptCliConfigTestUtility args) { 42 | final CleanRoomsDao cleanRoomsDao; 43 | cleanRoomsDao = CleanRoomsDaoTestUtility.generateMockDao(); 44 | when(cleanRoomsDao.getCollaborationDataEncryptionMetadata(any())).thenReturn(args.getClientSettings()); 45 | return EncryptMode.getApp(cleanRoomsDao).execute(args.toArrayWithoutMode()); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /c3r-cli/src/test/java/com/amazonaws/c3r/cli/MainEnvVarKeyInvalidTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.cli; 5 | 6 | import org.junit.jupiter.api.BeforeEach; 7 | import org.junit.jupiter.api.Test; 8 | import picocli.CommandLine; 9 | 10 | import static org.junit.jupiter.api.Assertions.assertNotEquals; 11 | 12 | /* 13 | * Tests specifically needing an invalid key in the environment 14 | * variable for the shared secret key. 15 | */ 16 | public class MainEnvVarKeyInvalidTest { 17 | private static final String ENC_INPUT_PATH = "../samples/csv/data_sample_without_quotes.csv"; 18 | 19 | private static final String SCHEMA_PATH = "../samples/schema/config_sample.json"; 20 | 21 | private static final String DEC_INPUT_PATH = "../samples/csv/marshalled_data_sample.csv"; 22 | 23 | private DecryptCliConfigTestUtility decArgs; 24 | 25 | private CommandLine decMain; 26 | 27 | private EncryptCliConfigTestUtility encArgs; 28 | 29 | private CommandLine encMain; 30 | 31 | public int runEncryptMainWithCliArgs() { 32 | return encMain.execute(encArgs.toArrayWithoutMode()); 33 | } 34 | 35 | public int runDecryptMainWithCliArgs() { 36 | return decMain.execute(decArgs.toArrayWithoutMode()); 37 | } 38 | 39 | @BeforeEach 40 | public void setup() { 41 | encArgs = EncryptCliConfigTestUtility.defaultDryRunTestArgs(ENC_INPUT_PATH, SCHEMA_PATH); 42 | encMain = EncryptMode.getApp(null); 43 | decArgs = DecryptCliConfigTestUtility.defaultDryRunTestArgs(DEC_INPUT_PATH); 44 | decMain = DecryptMode.getApp(); 45 | } 46 | 47 | @Test 48 | public void validateEncryptSecretKeyInvalidTest() { 49 | assertNotEquals(0, runEncryptMainWithCliArgs()); 50 | } 51 | 52 | @Test 53 | public void validateDecryptSecretKeyInvalidTest() { 54 | assertNotEquals(0, runDecryptMainWithCliArgs()); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /c3r-cli/src/test/java/com/amazonaws/c3r/io/schema/CsvSchemaGeneratorTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.io.schema; 5 | 6 | import com.amazonaws.c3r.exception.C3rRuntimeException; 7 | import com.amazonaws.c3r.utils.FileTestUtility; 8 | import com.amazonaws.c3r.utils.FileUtil; 9 | import org.junit.jupiter.api.Test; 10 | 11 | import java.io.IOException; 12 | import java.nio.file.Path; 13 | 14 | import static com.amazonaws.c3r.utils.GeneralTestUtility.DATA_SAMPLE_HEADERS; 15 | import static org.junit.jupiter.api.Assertions.assertEquals; 16 | import static org.junit.jupiter.api.Assertions.assertThrows; 17 | 18 | public class CsvSchemaGeneratorTest { 19 | 20 | private CsvSchemaGenerator getTestSchemaGenerator(final String file) throws IOException { 21 | final String output = FileTestUtility.resolve("schema.json").toString(); 22 | return CsvSchemaGenerator.builder() 23 | .inputCsvFile(file) 24 | .hasHeaders(true) 25 | .targetJsonFile(output) 26 | .overwrite(true) 27 | .build(); 28 | } 29 | 30 | @Test 31 | public void getSourceHeadersTest() throws IOException { 32 | assertEquals( 33 | DATA_SAMPLE_HEADERS, 34 | getTestSchemaGenerator(FileUtil.CURRENT_DIR + "/../samples/csv/data_sample_without_quotes.csv").getSourceHeaders()); 35 | } 36 | 37 | @Test 38 | public void getSourceColumnCountTest() throws IOException { 39 | assertEquals( 40 | DATA_SAMPLE_HEADERS.size(), 41 | getTestSchemaGenerator(FileUtil.CURRENT_DIR + "/../samples/csv/data_sample_without_quotes.csv").getSourceColumnCount()); 42 | } 43 | 44 | @Test 45 | public void emptyFileTest() throws IOException { 46 | final Path emptyCsvFile = FileTestUtility.createTempFile("empty", ".csv"); 47 | assertThrows(C3rRuntimeException.class, () -> 48 | getTestSchemaGenerator(emptyCsvFile.toString())); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /c3r-cli/src/test/java/com/amazonaws/c3r/utils/StringTestUtility.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.utils; 5 | 6 | import java.util.regex.Pattern; 7 | 8 | public final class StringTestUtility { 9 | 10 | private StringTestUtility() { 11 | } 12 | 13 | /** 14 | * Counts how many times a search string occurs (non-overlapping) in given string content. 15 | * 16 | * @param searchString String to search for 17 | * @param content Content to search in 18 | * @return The number of occurrences of the search string in the content. 19 | */ 20 | public static int countMatches(final String searchString, final String content) { 21 | return content.split(Pattern.quote(searchString), -1).length - 1; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /c3r-cli/src/test/java/com/amazonaws/c3r/utils/StringTestUtilityTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.utils; 5 | 6 | import org.junit.jupiter.api.Test; 7 | 8 | import static com.amazonaws.c3r.utils.StringTestUtility.countMatches; 9 | import static org.junit.jupiter.api.Assertions.assertEquals; 10 | 11 | public class StringTestUtilityTest { 12 | 13 | @Test 14 | public void countMatchesTest() { 15 | assertEquals(0, countMatches("a", "")); 16 | assertEquals(0, countMatches("a", "b")); 17 | assertEquals(1, countMatches("a", "a")); 18 | assertEquals(1, countMatches("a", "abcd")); 19 | assertEquals(3, countMatches("a", "abcdabcdabcd")); 20 | assertEquals(3, countMatches("aa", "aaabcdaaabcdaaabcd")); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /c3r-cli/src/test/java/com/amazonaws/c3r/utils/TimingResultTestUtility.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.utils; 5 | 6 | import com.amazonaws.c3r.config.ColumnType; 7 | import lombok.Builder; 8 | 9 | /** 10 | * Used to store performance testing metrics. 11 | */ 12 | @Builder 13 | public class TimingResultTestUtility { 14 | /** 15 | * Header names for timing results. 16 | */ 17 | public static final String[] HEADERS = { 18 | "Columns", 19 | "Rows", 20 | "Marshal Time (s)", 21 | "Unmarshal Time (s)", 22 | "Input Size (MB)", 23 | "Marshalled Size (MB)", 24 | "Unmarshalled Size (MB)", 25 | "Cleartext Columns", 26 | "Sealed Columns", 27 | "Fingerprint Columns", 28 | "Chars/Entry" 29 | }; 30 | 31 | /** 32 | * How many column types we are supporting. 33 | */ 34 | private static final int NUM_COL_TYPES = ColumnType.values().length; 35 | 36 | /** 37 | * Conversion factor for bytes to megabytes. 38 | */ 39 | private static final double MB = Math.pow(2, 20); 40 | 41 | /** 42 | * How many characters per entry in the input file. 43 | */ 44 | private Integer charsPerEntry; 45 | 46 | /** 47 | * Number of columns in the files. 48 | */ 49 | private Integer columnCount; 50 | 51 | /** 52 | * Number of rows in the files. 53 | */ 54 | private Long rowCount; 55 | 56 | /** 57 | * Size of original input file. 58 | */ 59 | private Long inputSizeBytes; 60 | 61 | /** 62 | * Time spent marshalling data. 63 | */ 64 | private Long marshalTimeSec; 65 | 66 | /** 67 | * Size of marshalled file. 68 | */ 69 | private Long marshalledSizeBytes; 70 | 71 | /** 72 | * Time spent unmarshalling data. 73 | */ 74 | private Long unmarshalTimeSec; 75 | 76 | /** 77 | * Size of the unmarshalled file. 78 | */ 79 | private Long unmarshalledSizeBytes; 80 | 81 | } -------------------------------------------------------------------------------- /c3r-cli/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | logs 5 | 6 | 7 | 9 | 10 | %d{dd MMM yyyy HH:mm:ss,SSS} [%p] (%t) %c: %m%n%ex 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | %d{dd MMM yyyy HH:mm:ss,SSS} [%p] (%t) %c: %m%n%ex 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/action/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * This package classes used to marshall (encrypt) and unmarshall (decrypt) data to and from the clean room for the various supported 6 | * data types. {@link com.amazonaws.c3r.action.RowMarshaller} handles the logic of marshalling data outside of anything having to do with 7 | * the actual data format and {@link com.amazonaws.c3r.action.RowUnmarshaller} does the same for unmarshalling. Each format specific class 8 | * handles file I/O and value creation only for that particular data type. 9 | * 10 | *

11 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 12 | * SPDX-License-Identifier: Apache-2.0 13 | */ 14 | package com.amazonaws.c3r.action; -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/cleanrooms/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * This package interfaces with AWS Clean Rooms to get collaboration information. 6 | * 7 | *

8 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 9 | * SPDX-License-Identifier: Apache-2.0 10 | */ 11 | @DefaultAnnotation(NonNull.class) 12 | package com.amazonaws.c3r.cleanrooms; 13 | 14 | import edu.umd.cs.findbugs.annotations.DefaultAnnotation; 15 | import lombok.NonNull; -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/config/ClientSettings.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.config; 5 | 6 | import lombok.Builder; 7 | import lombok.Value; 8 | 9 | import java.io.Serializable; 10 | 11 | /** 12 | * Contains clean room wide settings. 13 | */ 14 | @Value 15 | @Builder 16 | public class ClientSettings implements Serializable { 17 | /** 18 | * Whether cleartext columns are allowed. 19 | */ 20 | private boolean allowCleartext; 21 | 22 | /** 23 | * Whether duplicate values are allowed in a fingerprint column. 24 | * 25 | * @see ColumnType#FINGERPRINT 26 | */ 27 | private boolean allowDuplicates; 28 | 29 | /** 30 | * Whether fingerprint column names need to match on queries. 31 | * 32 | * @see ColumnType#FINGERPRINT 33 | */ 34 | private boolean allowJoinsOnColumnsWithDifferentNames; 35 | 36 | /** 37 | * Whether {@code null} values should be encrypted or left as {@code null}. 38 | */ 39 | private boolean preserveNulls; 40 | 41 | /** 42 | * Most permissive settings. 43 | * 44 | * @return ClientSettings with all flags set to `true` 45 | */ 46 | public static ClientSettings lowAssuranceMode() { 47 | return ClientSettings.builder() 48 | .allowCleartext(true) 49 | .allowDuplicates(true) 50 | .allowJoinsOnColumnsWithDifferentNames(true) 51 | .preserveNulls(true) 52 | .build(); 53 | } 54 | 55 | /** 56 | * Least permissive settings. 57 | * 58 | * @return ClientSettings with all flags set to `false` 59 | */ 60 | public static ClientSettings highAssuranceMode() { 61 | return ClientSettings.builder() 62 | .allowCleartext(false) 63 | .allowDuplicates(false) 64 | .allowJoinsOnColumnsWithDifferentNames(false) 65 | .preserveNulls(false) 66 | .build(); 67 | } 68 | } -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/config/ColumnType.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.config; 5 | 6 | import com.amazonaws.c3r.CleartextTransformer; 7 | import com.amazonaws.c3r.FingerprintTransformer; 8 | import com.amazonaws.c3r.SealedTransformer; 9 | 10 | import java.io.Serializable; 11 | 12 | /** 13 | * Differentiators for how a column is represented. 14 | */ 15 | public enum ColumnType implements Serializable { 16 | /** 17 | * Encrypted, meant to be used in the SELECT clause of an SQL query. 18 | */ 19 | SEALED("sealed", SealedTransformer.class), 20 | 21 | /** 22 | * HMACed, meant to be used in ON clauses of an SQL query. 23 | */ 24 | FINGERPRINT("fingerprint", FingerprintTransformer.class), 25 | 26 | /** 27 | * Cleartext, can be used in any clause of an SQL query. 28 | */ 29 | CLEARTEXT("cleartext", CleartextTransformer.class); 30 | 31 | /** 32 | * Associated transformer. 33 | */ 34 | private final Class transformerClass; 35 | 36 | /** 37 | * Formatted version of name. 38 | */ 39 | private final String name; 40 | 41 | /** 42 | * Associates column type with a specific transformer. 43 | * 44 | * @param name How the enum should be displayed when transformed to a string 45 | * @param clazz Name of transformer class 46 | * @see com.amazonaws.c3r.Transformer 47 | */ 48 | ColumnType(final String name, final Class clazz) { 49 | this.name = name; 50 | this.transformerClass = clazz; 51 | } 52 | 53 | /** 54 | * Get the type of transformer that should be used for the column type. 55 | * 56 | * @return Corresponding transformer 57 | */ 58 | public Class getTransformerType() { 59 | return transformerClass; 60 | } 61 | 62 | /** 63 | * {@inheritDoc} 64 | */ 65 | @Override 66 | public String toString() { 67 | return name; 68 | } 69 | } -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/config/PadType.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.config; 5 | 6 | import java.io.Serializable; 7 | 8 | /** 9 | * Differentiators for padding types. 10 | */ 11 | public enum PadType implements Serializable { 12 | /** 13 | * Values are not padded. 14 | */ 15 | NONE, 16 | 17 | /** 18 | * Values are padded to a user-specified {@code PAD_LENGTH}. 19 | */ 20 | FIXED, 21 | 22 | /** 23 | * Values are padded to {@code MAX_SIZE + PAD_LENGTH} where {@code MAX_SIZE} is the size of the 24 | * longest value in the column and {@code PAD_LENGTH} is user-specified. 25 | */ 26 | MAX 27 | } -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/config/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Classes that contain all the information needed to perform cryptographic computations on input data. 6 | * 7 | *

8 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 9 | * SPDX-License-Identifier: Apache-2.0 10 | */ 11 | package com.amazonaws.c3r.config; -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/data/CsvRow.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.data; 5 | 6 | import com.amazonaws.c3r.config.ColumnHeader; 7 | import com.amazonaws.c3r.internal.Nonce; 8 | import lombok.NonNull; 9 | 10 | /** 11 | * Row of CSV data. 12 | */ 13 | public final class CsvRow extends Row { 14 | /** 15 | * Create an empty CSV row. 16 | */ 17 | public CsvRow() { 18 | } 19 | 20 | /** 21 | * Create a fresh version of a CSV row. 22 | * 23 | * @param other Row to copy 24 | */ 25 | public CsvRow(final Row other) { 26 | other.forEach(this::putValue); 27 | } 28 | 29 | /** 30 | * {@inheritDoc} 31 | */ 32 | @Override 33 | public void putBytes(@NonNull final ColumnHeader column, final byte[] encodedValue) { 34 | putValue(column, new CsvValue(encodedValue)); 35 | } 36 | 37 | /** 38 | * {@inheritDoc} 39 | */ 40 | @Override 41 | public void putNonce(@NonNull final ColumnHeader nonceColumn, final Nonce nonce) { 42 | putBytes(nonceColumn, nonce.getBytes()); 43 | } 44 | 45 | /** 46 | * {@inheritDoc} 47 | */ 48 | @Override 49 | public Row clone() { 50 | final Row row = new CsvRow(); 51 | getHeaders().forEach(header -> row.putValue(header, getValue(header))); 52 | return row; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/data/ValueFactory.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.data; 5 | 6 | /** 7 | * Used to create values or containers of values for a particular data format. 8 | * 9 | * @param Data format 10 | */ 11 | public interface ValueFactory { 12 | /** 13 | * Takes a byte array of an encoded value and recreates the original value. 14 | * 15 | * @param bytes Encoded value 16 | * @return Recreated value 17 | */ 18 | T createValueFromEncodedBytes(byte[] bytes); 19 | 20 | /** 21 | * Takes a byte array of an encoded value and gets the bytes representing the internal value. 22 | * 23 | * @param bytes Encoded value 24 | * @return Bytes containing the value without metadata 25 | */ 26 | byte[] getValueBytesFromEncodedBytes(byte[] bytes); 27 | 28 | /** 29 | * Create an empty row to be populated by the callee. 30 | * 31 | * @return An empty Row for storing data in 32 | */ 33 | Row newRow(); 34 | } 35 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/data/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Implementation of {@link com.amazonaws.c3r.data.Value}, {@link com.amazonaws.c3r.data.Row}, and 6 | * {@link com.amazonaws.c3r.data.ValueFactory} for each supported data format. Also contains metadata 7 | * classes to store additional context for cryptographic operations on values. 8 | * 9 | *

10 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 11 | * SPDX-License-Identifier: Apache-2.0 12 | */ 13 | package com.amazonaws.c3r.data; -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/encryption/keys/DerivedRootEncryptionKey.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.encryption.keys; 5 | 6 | import javax.crypto.SecretKey; 7 | import javax.crypto.spec.SecretKeySpec; 8 | import java.nio.charset.StandardCharsets; 9 | 10 | /** 11 | * This class is a wrapper on SecretKeys. It adds type safety for ensuring that a DerivedRootEncryptionKey is never used for 12 | * encrypting individual data and that only a DerivedRootEncryptionKey may be used for generating {@link DerivedEncryptionKey} objects. 13 | */ 14 | public class DerivedRootEncryptionKey extends Key { 15 | /** 16 | * Creates the key to be used for all future key delegations. Note that this key and salt pair must be shared between all parties. 17 | * 18 | * @param secretKey The key to be used to initialize the root key 19 | * @param salt The salt to be used to initialize the root key 20 | */ 21 | public DerivedRootEncryptionKey(final SecretKey secretKey, final byte[] salt) { 22 | super(deriveRootEncryptionKey(secretKey, salt)); 23 | } 24 | 25 | /** 26 | * Creates the key to be used for all future key delegations. Note that this key and salt pair must be shared across all members of 27 | * the collaboration. 28 | * 29 | * @param secretKey The key to be used to initialize the root key 30 | * @param salt The salt to be used to initialize the root key 31 | * @return A key derived from the HMAC of the key provided 32 | */ 33 | private static SecretKey deriveRootEncryptionKey(final SecretKey secretKey, final byte[] salt) { 34 | final SaltedHkdf hkdf = new SaltedHkdf(secretKey, salt); 35 | return new SecretKeySpec(hkdf.deriveKey(KeyUtil.HKDF_INFO.getBytes(StandardCharsets.UTF_8), 36 | KeyUtil.SHARED_SECRET_KEY_BYTE_LENGTH), KeyUtil.KEY_ALG); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/encryption/keys/Key.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.encryption.keys; 5 | 6 | import com.amazonaws.c3r.exception.C3rIllegalArgumentException; 7 | 8 | import javax.crypto.SecretKey; 9 | 10 | /** 11 | * This class is a wrapper on SecretKeys, providing bare minimum validation on a SecretKey. The class is package private and may not be 12 | * instantiated. See child classes {@link DerivedRootEncryptionKey} and {@link DerivedEncryptionKey} for more info. 13 | */ 14 | abstract class Key implements SecretKey { 15 | /** 16 | * Symmetric key. 17 | */ 18 | protected final SecretKey secretKey; 19 | 20 | /** 21 | * Initialize symmetric key and validate the value. 22 | * 23 | * @param secretKey Symmetric key 24 | */ 25 | Key(final SecretKey secretKey) { 26 | this.secretKey = secretKey; 27 | validate(); 28 | } 29 | 30 | /** 31 | * Make sure key is not {@code null}. 32 | * 33 | * @throws C3rIllegalArgumentException If key is {@code null} 34 | */ 35 | public void validate() { 36 | if (secretKey == null) { 37 | throw new C3rIllegalArgumentException("The SecretKey must not be null."); 38 | } 39 | } 40 | 41 | /** 42 | * {@inheritDoc} 43 | */ 44 | @Override 45 | public String getAlgorithm() { 46 | return secretKey.getAlgorithm(); 47 | } 48 | 49 | /** 50 | * {@inheritDoc} 51 | */ 52 | @Override 53 | public String getFormat() { 54 | return secretKey.getFormat(); 55 | } 56 | 57 | /** 58 | * {@inheritDoc} 59 | */ 60 | @Override 61 | public byte[] getEncoded() { 62 | return secretKey.getEncoded(); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/encryption/keys/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Classes that generate the various keys used for different cryptographic operations. 6 | * 7 | *

8 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 9 | * SPDX-License-Identifier: Apache-2.0 10 | */ 11 | package com.amazonaws.c3r.encryption.keys; -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/encryption/materials/AbstractRawMaterials.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.encryption.materials; 5 | 6 | /** 7 | * Combines {@link DecryptionMaterials} and {@link EncryptionMaterials} into one class for managing all cryptographic operations. 8 | */ 9 | public abstract class AbstractRawMaterials implements DecryptionMaterials, EncryptionMaterials { 10 | } 11 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/encryption/materials/CryptographicMaterials.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.encryption.materials; 5 | 6 | /** 7 | * Top level interface for all the materials. Anything meant to be global to all the materials should be added here. 8 | */ 9 | public interface CryptographicMaterials { 10 | } -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/encryption/materials/DecryptionMaterials.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.encryption.materials; 5 | 6 | import com.amazonaws.c3r.encryption.keys.DerivedRootEncryptionKey; 7 | 8 | /** 9 | * Interface that specifies functions for decryption key usage. 10 | */ 11 | public interface DecryptionMaterials extends CryptographicMaterials { 12 | /** 13 | * Get the root decryption key in use. 14 | * 15 | * @return Decryption key 16 | */ 17 | DerivedRootEncryptionKey getRootDecryptionKey(); 18 | } -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/encryption/materials/EncryptionMaterials.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.encryption.materials; 5 | 6 | import com.amazonaws.c3r.encryption.keys.DerivedRootEncryptionKey; 7 | 8 | /** 9 | * Interface that specifies functions for encryption key usage. 10 | */ 11 | public interface EncryptionMaterials extends CryptographicMaterials { 12 | /** 13 | * Get the root encryption key in use. 14 | * 15 | * @return Encryption key 16 | */ 17 | DerivedRootEncryptionKey getRootEncryptionKey(); 18 | } -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/encryption/materials/SymmetricRawMaterials.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.encryption.materials; 5 | 6 | import com.amazonaws.c3r.encryption.keys.DerivedRootEncryptionKey; 7 | 8 | /** 9 | * Stores a symmetric root key, managing the encryption and decryption keys derived from it. 10 | */ 11 | public class SymmetricRawMaterials extends AbstractRawMaterials { 12 | /** 13 | * Symmetric key used for cryptographic operations. 14 | */ 15 | private final DerivedRootEncryptionKey cryptoKey; 16 | 17 | /** 18 | * Stores root symmetric encryption key for managing encryption/decryption operations. 19 | * 20 | * @param encryptionKey Symmetric key 21 | */ 22 | public SymmetricRawMaterials(final DerivedRootEncryptionKey encryptionKey) { 23 | this.cryptoKey = encryptionKey; 24 | } 25 | 26 | /** 27 | * {@inheritDoc} 28 | */ 29 | @Override 30 | public DerivedRootEncryptionKey getRootEncryptionKey() { 31 | return cryptoKey; 32 | } 33 | 34 | /** 35 | * {@inheritDoc} 36 | */ 37 | @Override 38 | public DerivedRootEncryptionKey getRootDecryptionKey() { 39 | return cryptoKey; 40 | } 41 | } -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/encryption/materials/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Classes for managing cryptographic keys. 6 | * 7 | *

8 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 9 | * SPDX-License-Identifier: Apache-2.0 10 | */ 11 | package com.amazonaws.c3r.encryption.materials; -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/encryption/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Cryptographic functions for C3R. 6 | * 7 | *

8 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 9 | * SPDX-License-Identifier: Apache-2.0 10 | */ 11 | package com.amazonaws.c3r.encryption; -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/encryption/providers/SymmetricStaticProvider.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.encryption.providers; 5 | 6 | import com.amazonaws.c3r.encryption.EncryptionContext; 7 | import com.amazonaws.c3r.encryption.keys.DerivedRootEncryptionKey; 8 | import com.amazonaws.c3r.encryption.materials.DecryptionMaterials; 9 | import com.amazonaws.c3r.encryption.materials.EncryptionMaterials; 10 | import com.amazonaws.c3r.encryption.materials.SymmetricRawMaterials; 11 | 12 | import javax.crypto.SecretKey; 13 | 14 | /** 15 | * This EncryptionMaterialsProvider may be used when a SecretKey/salt has been shared out of band or is otherwise provided 16 | * programmatically. Data encrypted with this EncryptionMaterialsProvider may be decrypted with the same symmetric key and salt provided at 17 | * construction. 18 | */ 19 | public class SymmetricStaticProvider implements EncryptionMaterialsProvider { 20 | /** 21 | * Implements a symmetric key cryptographic algorithm. 22 | */ 23 | private final SymmetricRawMaterials materials; 24 | 25 | /** 26 | * Creates a handler for symmetric keys, using {@code encryptionKey} and {@code salt} to generate sub-keys as needed. 27 | * 28 | * @param encryptionKey the key materials for the root key 29 | * @param salt the salt for the root key 30 | */ 31 | public SymmetricStaticProvider(final SecretKey encryptionKey, final byte[] salt) { 32 | materials = new SymmetricRawMaterials(new DerivedRootEncryptionKey(encryptionKey, salt)); 33 | } 34 | 35 | 36 | /** 37 | * Returns the {@code encryptionKey} provided to the constructor. 38 | */ 39 | @Override 40 | public DecryptionMaterials getDecryptionMaterials(final EncryptionContext context) { 41 | return new SymmetricRawMaterials(materials.getRootEncryptionKey()); 42 | } 43 | 44 | /** 45 | * Returns the {@code encryptionKey} provided to the constructor. 46 | */ 47 | @Override 48 | public EncryptionMaterials getEncryptionMaterials(final EncryptionContext context) { 49 | return new SymmetricRawMaterials(materials.getRootEncryptionKey()); 50 | } 51 | 52 | /** 53 | * Does nothing. 54 | */ 55 | @Override 56 | public void refresh() { 57 | // Do nothing 58 | } 59 | } -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/encryption/providers/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Cryptographic implementations for key generation algorithms. 6 | * 7 | *

8 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 9 | * SPDX-License-Identifier: Apache-2.0 10 | */ 11 | package com.amazonaws.c3r.encryption.providers; -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/exception/C3rIllegalArgumentException.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.exception; 5 | 6 | /** 7 | * Like {@link IllegalArgumentException}, but contains an error message 8 | * that is always safe to be printed/logged. 9 | * 10 | * @see C3rRuntimeException 11 | */ 12 | public class C3rIllegalArgumentException extends C3rRuntimeException { 13 | /** 14 | * Construct an unchecked runtime exception for an invalid method parameter value. 15 | * 16 | * @param message Safe error message text for printing and logging 17 | */ 18 | public C3rIllegalArgumentException(final String message) { 19 | super(message); 20 | } 21 | 22 | /** 23 | * Construct an unchecked runtime exception for an invalid method parameter value. 24 | * 25 | * @param message Safe error message text for printing and logging 26 | * @param cause Original error that may not be safe to print or log in case the user has a higher level of logging enabled 27 | */ 28 | public C3rIllegalArgumentException(final String message, final Throwable cause) { 29 | super(message, cause); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/exception/C3rRuntimeException.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.exception; 5 | 6 | /** 7 | * Like {@link RuntimeException}, but contains an error message 8 | * that is always safe to be printed/logged. 9 | */ 10 | public class C3rRuntimeException extends RuntimeException { 11 | /** 12 | * Construct an unchecked runtime exception. 13 | * 14 | * @param message Safe error message for printing and logging 15 | */ 16 | public C3rRuntimeException(final String message) { 17 | super(message); 18 | } 19 | 20 | /** 21 | * Construct an unchecked runtime exception. 22 | * 23 | * @param message Safe error message for printing and logging 24 | * @param cause Original error that may not be safe to print or log in case the user has a higher level of logging enabled 25 | */ 26 | public C3rRuntimeException(final String message, final Throwable cause) { 27 | super(message, cause); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/exception/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Contains exceptions used by C3R that will only have information in the message portion that is safe for printing and logging. 6 | * 7 | *

8 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 9 | * SPDX-License-Identifier: Apache-2.0 10 | */ 11 | package com.amazonaws.c3r.exception; -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/internal/AdditionalAuthenticatedData.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.internal; 5 | 6 | /** 7 | * Stores AAD for use during operation to confirm origin and authenticity of data. 8 | */ 9 | public class AdditionalAuthenticatedData { 10 | /** 11 | * AAD value. 12 | */ 13 | private byte[] bytes; 14 | 15 | /** 16 | * Stores a value used to verify data is from the expected source. 17 | * 18 | * @param bytes Value to identify your data as authentic 19 | */ 20 | public AdditionalAuthenticatedData(final byte[] bytes) { 21 | if (bytes != null) { 22 | this.bytes = bytes.clone(); 23 | } 24 | } 25 | 26 | /** 27 | * Get the AAD value. 28 | * 29 | * @return AAD 30 | */ 31 | public byte[] getBytes() { 32 | if (bytes != null) { 33 | return bytes.clone(); 34 | } 35 | return null; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/internal/Nonce.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.internal; 5 | 6 | import com.amazonaws.c3r.exception.C3rIllegalArgumentException; 7 | 8 | import java.security.SecureRandom; 9 | 10 | /** 11 | * Random number used for various cryptographic operations. 12 | */ 13 | public class Nonce { 14 | /** 15 | * Length of nonce in bytes. 16 | */ 17 | public static final int NONCE_BYTE_LENGTH = 32; 18 | 19 | /** 20 | * Use the most secure CSPRNG available on the system to generate random numbers. 21 | */ 22 | private static final SecureRandom NONCE_GENERATOR = new SecureRandom(); 23 | 24 | /** 25 | * Random number. 26 | */ 27 | private byte[] bytes; 28 | 29 | /** 30 | * Store a copy of an already existing nonce. 31 | * 32 | * @param bytes Random value 33 | */ 34 | public Nonce(final byte[] bytes) { 35 | if (bytes != null) { 36 | this.bytes = bytes.clone(); 37 | } 38 | 39 | validate(); 40 | } 41 | 42 | /** 43 | * Generates a cryptographically secure nonce of {@value Nonce#NONCE_BYTE_LENGTH} bytes. 44 | * 45 | * @return A nonce consisting of random bytes 46 | */ 47 | public static Nonce nextNonce() { 48 | final byte[] nonceBytes = new byte[Nonce.NONCE_BYTE_LENGTH]; 49 | NONCE_GENERATOR.nextBytes(nonceBytes); 50 | return new Nonce(nonceBytes); 51 | } 52 | 53 | /** 54 | * Get a copy of the value. 55 | * 56 | * @return Unique copy of nonce 57 | */ 58 | public byte[] getBytes() { 59 | return bytes.clone(); 60 | } 61 | 62 | /** 63 | * Validates that the nonce is not {@code null} and is of the required length {@value #NONCE_BYTE_LENGTH} bytes. 64 | * 65 | * @throws C3rIllegalArgumentException If the nonce does not meet requirements 66 | */ 67 | private void validate() { 68 | if (bytes == null) { 69 | throw new C3rIllegalArgumentException("A Nonce may not be null."); 70 | } else if (bytes.length != NONCE_BYTE_LENGTH) { 71 | throw new C3rIllegalArgumentException("An Nonce must be " + NONCE_BYTE_LENGTH + " bytes in length, but was " + bytes.length 72 | + " bytes."); 73 | 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/internal/Validatable.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.internal; 5 | 6 | /** 7 | * To be implemented by any class that requires a validation step after being constructed/deserialized. 8 | */ 9 | public interface Validatable { 10 | /** 11 | * Checks type instance for validity. Used to ensure consistent construction of data types between GSON and Java generated instances. 12 | */ 13 | void validate(); 14 | } 15 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/internal/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Classes that are only used internally in C3R. These are in support of prepping data for a cryptographic operation or 6 | * contain values used to execute those operations. 7 | * 8 | *

9 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 10 | * SPDX-License-Identifier: Apache-2.0 11 | */ 12 | package com.amazonaws.c3r.internal; -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/io/FileFormat.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.io; 5 | 6 | import java.util.Arrays; 7 | import java.util.Map; 8 | import java.util.function.Function; 9 | import java.util.stream.Collectors; 10 | 11 | /** 12 | * Provides support for file format type information such as file extensions. 13 | */ 14 | public enum FileFormat { 15 | /** 16 | * CSV formatted file. 17 | */ 18 | CSV(".csv"), 19 | /** 20 | * Parquet formatted file. 21 | */ 22 | PARQUET(".parquet"); 23 | 24 | /** 25 | * Lookup table to map string extension to enum representation of extension. 26 | */ 27 | private static final Map EXTENSIONS = Arrays.stream(FileFormat.values()).collect( 28 | Collectors.toMap( 29 | fmt -> fmt.extension, 30 | Function.identity() 31 | )); 32 | /** 33 | * String containing the file extension. 34 | */ 35 | private final String extension; 36 | 37 | /** 38 | * Construct a FileFormat with the given file extension. 39 | * 40 | * @param extension Supported file type extension to use 41 | */ 42 | FileFormat(final String extension) { 43 | this.extension = extension; 44 | } 45 | 46 | /** 47 | * Check and see if the input file name has an extension specifying the file type. 48 | * 49 | * @param fileName Input file name 50 | * @return Supported data type or {@code null} 51 | */ 52 | public static FileFormat fromFileName(final String fileName) { 53 | final int extensionStart = fileName.lastIndexOf('.'); 54 | if (extensionStart < 0) { 55 | return null; 56 | } 57 | return EXTENSIONS.get(fileName.substring(extensionStart).toLowerCase()); 58 | 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/io/RowWriter.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.io; 5 | 6 | import com.amazonaws.c3r.config.ColumnHeader; 7 | import com.amazonaws.c3r.data.Row; 8 | import com.amazonaws.c3r.data.Value; 9 | import lombok.NonNull; 10 | 11 | import java.util.Collection; 12 | 13 | /** 14 | * Resource for storing record entries to some store/backend. 15 | * 16 | * @param Specific data type that will be written 17 | */ 18 | public interface RowWriter { 19 | /** 20 | * Gets the headers for the output file. 21 | * 22 | * @return The ColumnHeaders of the output file 23 | */ 24 | Collection getHeaders(); 25 | 26 | /** 27 | * Write a record to the store. 28 | * 29 | * @param row Row to write with columns mapped to respective values 30 | */ 31 | void writeRow(@NonNull Row row); 32 | 33 | /** 34 | * Close this record source so the resource is no longer in use. 35 | */ 36 | void close(); 37 | 38 | /** 39 | * Flush the write buffer. 40 | */ 41 | void flush(); 42 | 43 | /** 44 | * Gets the target file name. 45 | * 46 | * @return Name of target file 47 | */ 48 | String getTargetName(); 49 | } 50 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/io/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Support for reading and writing rows of data for various data sources. 6 | * 7 | *

8 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 9 | * SPDX-License-Identifier: Apache-2.0 10 | */ 11 | package com.amazonaws.c3r.io; -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/io/sql/SqlTable.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.io.sql; 5 | 6 | import com.amazonaws.c3r.exception.C3rRuntimeException; 7 | import lombok.Getter; 8 | 9 | import java.io.File; 10 | import java.sql.Connection; 11 | import java.sql.DriverManager; 12 | import java.sql.SQLException; 13 | 14 | /** 15 | * A SQL connection and the underlying database file. 16 | */ 17 | @Getter 18 | public class SqlTable { 19 | /** 20 | * Connection to the SQL database for this session. 21 | */ 22 | private final Connection connection; 23 | 24 | /** 25 | * SQL database file. 26 | */ 27 | private final File databaseFile; 28 | 29 | /** 30 | * Creates a connection to a SQL database using the file as the database source. 31 | * 32 | * @param databaseFile File to use as a SQL database 33 | * @throws C3rRuntimeException If there's an error accessing file 34 | */ 35 | public SqlTable(final File databaseFile) { 36 | try { 37 | this.connection = DriverManager.getConnection("jdbc:sqlite:" + databaseFile.getAbsolutePath()); 38 | } catch (SQLException e) { 39 | throw new C3rRuntimeException("Could not access SQL database.", e); 40 | } 41 | this.databaseFile = databaseFile; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/io/sql/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Support classes for managing row-based access to a SQL database. 6 | * 7 | *

8 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 9 | * SPDX-License-Identifier: Apache-2.0 10 | * 11 | * @see com.amazonaws.c3r.io.SqlRowReader 12 | * @see com.amazonaws.c3r.io.SqlRowWriter 13 | */ 14 | package com.amazonaws.c3r.io.sql; -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/json/ColumnHeaderTypeAdapter.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.json; 5 | 6 | import com.amazonaws.c3r.config.ColumnHeader; 7 | import com.amazonaws.c3r.exception.C3rRuntimeException; 8 | import com.google.gson.TypeAdapter; 9 | import com.google.gson.stream.JsonReader; 10 | import com.google.gson.stream.JsonToken; 11 | import com.google.gson.stream.JsonWriter; 12 | 13 | import java.io.IOException; 14 | 15 | /** 16 | * Handles the serialization/deserialization of ColumnHeader names. Allows for case-insensitivity. 17 | */ 18 | public class ColumnHeaderTypeAdapter extends TypeAdapter { 19 | /** 20 | * Serialize {@link ColumnHeader} object to a string and send to {@code out}. 21 | * 22 | * @param out Formatted JSON output to add to 23 | * @param value The column header name to write 24 | * @throws C3rRuntimeException If there's an error writing to output 25 | */ 26 | @Override 27 | public void write(final JsonWriter out, final ColumnHeader value) { 28 | try { 29 | if (value == null) { 30 | out.nullValue(); 31 | } else { 32 | out.value(value.toString()); 33 | } 34 | } catch (IOException e) { 35 | throw new C3rRuntimeException("Error writing to output", e); 36 | } 37 | } 38 | 39 | /** 40 | * Read in a JSON value and attempt to deserialize it as a {@link ColumnHeader}. 41 | * 42 | * @param in Source to read value from 43 | * @return The value parsed as a header name 44 | * @throws C3rRuntimeException If there's an error reading from source 45 | */ 46 | @Override 47 | public ColumnHeader read(final JsonReader in) { 48 | in.setLenient(true); 49 | try { 50 | if (in.peek() == JsonToken.NULL) { 51 | in.nextNull(); 52 | return null; 53 | } else { 54 | return new ColumnHeader(in.nextString()); 55 | } 56 | } catch (IOException e) { 57 | throw new C3rRuntimeException("Error reading from input", e); 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/json/ColumnTypeTypeAdapter.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.json; 5 | 6 | import com.amazonaws.c3r.config.ColumnType; 7 | import com.amazonaws.c3r.exception.C3rRuntimeException; 8 | import com.google.gson.TypeAdapter; 9 | import com.google.gson.stream.JsonReader; 10 | import com.google.gson.stream.JsonToken; 11 | import com.google.gson.stream.JsonWriter; 12 | 13 | import java.io.IOException; 14 | 15 | /** 16 | * Handles the serialization/deserialization of ColumnTypes. Allows for case-insensitivity. 17 | */ 18 | public class ColumnTypeTypeAdapter extends TypeAdapter { 19 | /** 20 | * Serialize {@link ColumnType} object to a string and send to {@code out}. 21 | * 22 | * @param out Stream of values written so far 23 | * @param value the Java object to write 24 | * @throws C3rRuntimeException If there's an error writing to output 25 | */ 26 | @Override 27 | public void write(final JsonWriter out, final ColumnType value) { 28 | try { 29 | if (value == null) { 30 | out.nullValue(); 31 | } else { 32 | out.value(value.toString()); 33 | } 34 | } catch (IOException e) { 35 | throw new C3rRuntimeException("Error writing to output.", e); 36 | } 37 | } 38 | 39 | /** 40 | * Read in a JSON value and attempt to deserialize it as a {@link ColumnType}. 41 | * 42 | * @param in Stream of tokenized JSON values 43 | * @return Type of column transform to use 44 | * @throws C3rRuntimeException If there's an error reading from source 45 | */ 46 | @Override 47 | public ColumnType read(final JsonReader in) { 48 | try { 49 | if (in.peek() == JsonToken.NULL) { 50 | in.nextNull(); 51 | return null; 52 | } else { 53 | return ColumnType.valueOf(in.nextString().trim().toUpperCase()); 54 | } 55 | } catch (IOException e) { 56 | throw new C3rRuntimeException("Error reading from input.", e); 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/json/PadTypeTypeAdapter.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.json; 5 | 6 | import com.amazonaws.c3r.config.PadType; 7 | import com.amazonaws.c3r.exception.C3rRuntimeException; 8 | import com.google.gson.TypeAdapter; 9 | import com.google.gson.stream.JsonReader; 10 | import com.google.gson.stream.JsonToken; 11 | import com.google.gson.stream.JsonWriter; 12 | 13 | import java.io.IOException; 14 | 15 | /** 16 | * Handles the serialization/deserialization of PadTypes. Allows for case-insensitivity. 17 | */ 18 | public class PadTypeTypeAdapter extends TypeAdapter { 19 | /** 20 | * Serialize {@link PadType} object to a string and send to {@code out}. 21 | * 22 | * @param out Output stream of formatted JSON data 23 | * @param value Pad type 24 | * @throws C3rRuntimeException If there's an error writing to output 25 | */ 26 | @Override 27 | public void write(final JsonWriter out, final PadType value) { 28 | try { 29 | if (value == null) { 30 | out.nullValue(); 31 | } else { 32 | out.value(value.name()); 33 | } 34 | } catch (IOException e) { 35 | throw new C3rRuntimeException("Unable to write to output.", e); 36 | } 37 | } 38 | 39 | /** 40 | * Read in a JSON value and attempt to deserialize it as a {@link PadType}. 41 | * 42 | * @param in Input stream of tokenized JSON 43 | * @return Pad type specified 44 | * @throws C3rRuntimeException If there's an error reading from source 45 | */ 46 | @Override 47 | public PadType read(final JsonReader in) { 48 | try { 49 | if (in.peek() == JsonToken.NULL) { 50 | in.nextNull(); 51 | return null; 52 | } else { 53 | return PadType.valueOf(in.nextString().toUpperCase()); 54 | } 55 | } catch (IOException e) { 56 | throw new C3rRuntimeException("Error reading from input.", e); 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/json/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Custom adapters and interfaces for importing schema information from JSON into the runtime environment. These include adapters for 6 | * types that can't be automatically serialized and deserialized plus interfaces to run standard steps on created objects. 7 | * 8 | *

9 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 10 | * SPDX-License-Identifier: Apache-2.0 11 | */ 12 | package com.amazonaws.c3r.json; -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Handles the transformation of cleartext data into encrypted data. 6 | * 7 | *

8 | * This package takes in the settings for a clean room, a data file and then preforms the necessary transformations to create the 9 | * encrypted output. The row marshallers and unmarshallers (see {@link com.amazonaws.c3r.action}) are the main entry point into the SDK. 10 | * To encrypt data, create a new instance of a {@link com.amazonaws.c3r.action.RowMarshaller} for the particular data format you are using, 11 | * then calling {@link com.amazonaws.c3r.action.RowMarshaller#marshal()} and {@link com.amazonaws.c3r.action.RowMarshaller#close()} will 12 | * transform the cleartext data according to the schema into a file containing the encrypted data. To decrypt data, an instance of a 13 | * {@link com.amazonaws.c3r.action.RowUnmarshaller} for the particular data type is created, then 14 | * {@link com.amazonaws.c3r.action.RowUnmarshaller#unmarshal()} and {@link com.amazonaws.c3r.action.RowUnmarshaller#close()} are called. 15 | * 16 | *

17 | * The settings are stored in an instance of {@link com.amazonaws.c3r.config.EncryptConfig} or 18 | * {@link com.amazonaws.c3r.config.DecryptConfig} for the respective mode of operation. The schema information is kept in an instance of 19 | * the {@link com.amazonaws.c3r.config.TableSchema} which is backed by several implementations. Between the configuration and schema 20 | * classes, the row marshallers and unmarshallers will have the information they need to do cryptographic transforms. 21 | * 22 | *

23 | * The rest of the packages and classes inside this package are in support of these top level classes. Classes in the package 24 | * {@code com.amazonaws.config} will be used in the course of creating the cryptographic configurations but the remaining classes are not 25 | * meant to be used directly for development as they may change at will. 26 | * 27 | *

28 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 29 | * SPDX-License-Identifier: Apache-2.0 30 | */ 31 | package com.amazonaws.c3r; 32 | 33 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/utils/C3rSdkProperties.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.utils; 5 | 6 | import software.amazon.awssdk.core.ApiName; 7 | 8 | /** 9 | * Utility class for storing C3R-wide constants. 10 | */ 11 | public final class C3rSdkProperties { 12 | 13 | /** 14 | * C3R version. 15 | */ 16 | public static final String VERSION = "3.0.5"; 17 | 18 | /** 19 | * C3R SDK user agent. 20 | */ 21 | public static final ApiName API_NAME = ApiName.builder().name("c3r-sdk").version(VERSION).build(); 22 | 23 | /** 24 | * Hidden utility class constructor. 25 | */ 26 | private C3rSdkProperties() { 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/java/com/amazonaws/c3r/utils/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Utility classes that contain commonly used functionality across components. 6 | * 7 | *

8 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 9 | * SPDX-License-Identifier: Apache-2.0 10 | */ 11 | package com.amazonaws.c3r.utils; -------------------------------------------------------------------------------- /c3r-sdk-core/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | logs 5 | 6 | 7 | 9 | 10 | %d{dd MMM yyyy HH:mm:ss,SSS} [%p] (%t) %c: %m%n%ex 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | %d{dd MMM yyyy HH:mm:ss,SSS} [%p] (%t) %c: %m%n%ex 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/CleartextTransformerTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r; 5 | 6 | import com.amazonaws.c3r.exception.C3rRuntimeException; 7 | import org.junit.jupiter.api.BeforeEach; 8 | import org.junit.jupiter.api.Test; 9 | 10 | import java.nio.charset.StandardCharsets; 11 | import java.util.Arrays; 12 | 13 | import static org.junit.jupiter.api.Assertions.assertArrayEquals; 14 | import static org.junit.jupiter.api.Assertions.assertFalse; 15 | import static org.junit.jupiter.api.Assertions.assertNull; 16 | import static org.junit.jupiter.api.Assertions.assertThrows; 17 | 18 | public class CleartextTransformerTest { 19 | 20 | private CleartextTransformer transformer; 21 | 22 | @BeforeEach 23 | public void setup() { 24 | transformer = new CleartextTransformer(); 25 | } 26 | 27 | @Test 28 | public void marshalTest() { 29 | final byte[] cleartext = "Some cleartext".getBytes(StandardCharsets.UTF_8); 30 | assertArrayEquals(cleartext, transformer.marshal(cleartext, null)); 31 | } 32 | 33 | @Test 34 | public void marshalledValueTooLongTest() { 35 | final byte[] cleartext = new byte[Transformer.MAX_GLUE_STRING_BYTES + 1]; 36 | assertThrows(C3rRuntimeException.class, () -> transformer.marshal(cleartext, null)); 37 | } 38 | 39 | @Test 40 | public void marshalNullTest() { 41 | assertNull(transformer.marshal(null, null)); 42 | } 43 | 44 | @Test 45 | public void unmarshalTest() { 46 | final byte[] ciphertext = "Some ciphertext".getBytes(StandardCharsets.UTF_8); 47 | assertArrayEquals(ciphertext, transformer.unmarshal(ciphertext)); 48 | } 49 | 50 | @Test 51 | public void unmarshalNullTest() { 52 | assertNull(transformer.unmarshal(null)); 53 | } 54 | 55 | @Test 56 | public void getEncryptionDescriptorTest() { 57 | assertThrows(C3rRuntimeException.class, () -> transformer.getEncryptionDescriptor()); 58 | } 59 | 60 | @Test 61 | public void getVersionImmutableTest() { 62 | final byte[] version = transformer.getVersion(); 63 | Arrays.fill(version, (byte) 0); 64 | assertFalse(Arrays.equals(transformer.getVersion(), version)); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/action/CsvRowUnmarshallerTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.action; 5 | 6 | import com.amazonaws.c3r.config.DecryptConfig; 7 | import com.amazonaws.c3r.exception.C3rIllegalArgumentException; 8 | import com.amazonaws.c3r.io.FileFormat; 9 | import com.amazonaws.c3r.utils.FileTestUtility; 10 | import org.junit.jupiter.api.Test; 11 | 12 | import java.io.IOException; 13 | 14 | import static com.amazonaws.c3r.utils.GeneralTestUtility.TEST_CONFIG_DATA_SAMPLE; 15 | import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; 16 | import static org.junit.jupiter.api.Assertions.assertThrows; 17 | 18 | public class CsvRowUnmarshallerTest { 19 | 20 | @Test 21 | public void validateRejectNonCsvFormatTest() throws IOException { 22 | final String output = FileTestUtility.resolve("endToEndMarshalOut.unknown").toString(); 23 | final var configBuilder = DecryptConfig.builder() 24 | .sourceFile(TEST_CONFIG_DATA_SAMPLE.getInput()) 25 | .targetFile(output) 26 | .fileFormat(FileFormat.CSV) 27 | .secretKey(TEST_CONFIG_DATA_SAMPLE.getKey()) 28 | .salt(TEST_CONFIG_DATA_SAMPLE.getSalt()) 29 | .overwrite(true); 30 | 31 | assertThrows(C3rIllegalArgumentException.class, () -> 32 | CsvRowUnmarshaller.newInstance(configBuilder.fileFormat(FileFormat.PARQUET).build())); 33 | assertDoesNotThrow(() -> 34 | CsvRowUnmarshaller.newInstance(configBuilder.fileFormat(FileFormat.CSV).build())); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/config/ConfigTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.config; 5 | 6 | import com.amazonaws.c3r.utils.FileTestUtility; 7 | import com.amazonaws.c3r.utils.FileUtil; 8 | import org.junit.jupiter.api.Test; 9 | 10 | import java.io.File; 11 | import java.io.IOException; 12 | import java.nio.file.Path; 13 | 14 | import static com.amazonaws.c3r.config.Config.getDefaultTargetFile; 15 | import static org.junit.jupiter.api.Assertions.assertEquals; 16 | import static org.junit.jupiter.api.Assertions.assertNotNull; 17 | import static org.junit.jupiter.api.Assertions.assertNull; 18 | import static org.junit.jupiter.api.Assertions.assertTrue; 19 | 20 | public class ConfigTest { 21 | 22 | @Test 23 | public void defaultTargetFileWithExtensionTest() { 24 | assertEquals("input.out.csv", getDefaultTargetFile("input.csv")); 25 | assertEquals("input.misc.out.csv", getDefaultTargetFile("input.misc.csv")); 26 | } 27 | 28 | @Test 29 | public void getDefaultTargetFileTest() throws IOException { 30 | final Path sourceFile = FileTestUtility.resolve("sourceFile.csv"); 31 | 32 | final String defaultTargetFile = Config.getDefaultTargetFile(sourceFile.toFile().getAbsolutePath()); 33 | 34 | // assert sourceFile directory is stripped and targetFile is associated with the working directory. 35 | final File targetFile = new File(defaultTargetFile); 36 | assertNotNull(sourceFile.getParent()); 37 | assertNull(targetFile.getParentFile()); 38 | assertTrue(targetFile.getAbsolutePath().contains(FileUtil.CURRENT_DIR)); 39 | } 40 | 41 | @Test 42 | public void defaultTargetFileWithoutExtensionTest() { 43 | assertEquals("input.out", getDefaultTargetFile("input")); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/config/PadTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.config; 5 | 6 | import com.amazonaws.c3r.exception.C3rIllegalArgumentException; 7 | import org.junit.jupiter.api.Test; 8 | import org.mockito.internal.util.reflection.ReflectionMemberAccessor; 9 | 10 | import java.lang.reflect.Field; 11 | 12 | import static org.junit.jupiter.api.Assertions.assertEquals; 13 | import static org.junit.jupiter.api.Assertions.assertNull; 14 | import static org.junit.jupiter.api.Assertions.assertThrows; 15 | import static org.junit.jupiter.api.Assertions.assertThrowsExactly; 16 | import static org.mockito.Mockito.doCallRealMethod; 17 | import static org.mockito.Mockito.mock; 18 | 19 | public class PadTest { 20 | 21 | @Test 22 | public void defaultPadTypeNoneTest() { 23 | final Pad pad = Pad.builder().build(); 24 | assertNull(pad.getType()); 25 | assertNull(pad.getLength()); 26 | } 27 | 28 | @Test 29 | public void padLengthPadTypeMaxTest() { 30 | assertThrows(C3rIllegalArgumentException.class, 31 | () -> Pad.builder().type(PadType.MAX).build()); 32 | } 33 | 34 | @Test 35 | public void padLengthPadTypeFixedTest() { 36 | assertThrows(C3rIllegalArgumentException.class, 37 | () -> Pad.builder().type(PadType.FIXED).build()); 38 | } 39 | 40 | @Test 41 | public void padLengthTest() { 42 | final Pad pad = Pad.builder().type(PadType.MAX).length(10).build(); 43 | assertEquals(10, pad.getLength()); 44 | } 45 | 46 | @Test 47 | public void badPadTest() throws NoSuchFieldException, IllegalAccessException { 48 | final Pad p = mock(Pad.class); 49 | doCallRealMethod().when(p).validate(); 50 | final var rma = new ReflectionMemberAccessor(); 51 | final Field type = Pad.class.getDeclaredField("type"); 52 | final Field length = Pad.class.getDeclaredField("length"); 53 | rma.set(type, p, null); 54 | rma.set(length, p, 10); 55 | final Exception e = assertThrowsExactly(C3rIllegalArgumentException.class, p::validate); 56 | assertEquals("A pad type is required if a pad length is specified but only a pad length was provided.", e.getMessage()); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/config/TableSchemaCommonTestInterface.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.config; 5 | 6 | // Defines a common set of tests all schema types should implement. 7 | public interface TableSchemaCommonTestInterface { 8 | // Verify that headers are correctly autogenerated as needed. 9 | void verifyAutomaticHeaderConstructionTest(); 10 | 11 | // Verify one source column can be mapped to multiple output columns as long as they have unique target header names. 12 | void oneSourceToManyTargetsTest(); 13 | 14 | // Verify that is the same target header name is used more than once in a schema, it fails validation. 15 | void repeatedTargetFailsTest(); 16 | 17 | // Validate that the column is null the schema is not accepted. 18 | void validateNullColumnValueIsRejectedTest(); 19 | 20 | // Validate that if the output column list is empty, the schema is not accepted. 21 | void validateEmptyColumnValueIsRejectedTest(); 22 | 23 | // Test the implementation of equals and hash to make sure results are as expected. 24 | void equalsAndHashTest(); 25 | 26 | // Verify the header row can't be the wrong value and the schema will still work. 27 | void verifyHeaderRowValueTest(); 28 | 29 | // Check the results from {@code getColumns()} matches the ones used in the schema and not the ones in the file. 30 | void verifyColumnsInResultsTest(); 31 | 32 | // Check that {@code getColumns()} returns properly validated columns. 33 | void verifyGetColumnsTest() throws IllegalAccessException, NoSuchFieldException; 34 | 35 | // Make sure the various states of unspecified headers are correct for the child implementation. 36 | void verifyGetUnspecifiedHeadersReturnValueTest() throws IllegalAccessException, NoSuchFieldException; 37 | 38 | // Make sure child implementation rejects all invalid headers for its type. 39 | void verifyChildSpecificInvalidHeaderConstructionTest(); 40 | 41 | // Check the child specific {@code verification} function to be sure it works as expected. 42 | void checkClassSpecificVerificationTest() throws IllegalAccessException, NoSuchFieldException; 43 | } 44 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/data/CsvValueTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.data; 5 | 6 | import org.junit.jupiter.api.Test; 7 | 8 | import java.nio.charset.StandardCharsets; 9 | 10 | import static org.junit.jupiter.api.Assertions.assertArrayEquals; 11 | import static org.junit.jupiter.api.Assertions.assertEquals; 12 | import static org.junit.jupiter.api.Assertions.assertNull; 13 | 14 | public class CsvValueTest { 15 | @Test 16 | public void stringConstructorNullTest() { 17 | assertNull(new CsvValue((String) null).getBytes()); 18 | assertNull(new CsvValue((byte[]) null).getBytes()); 19 | assertEquals(0, new CsvValue((byte[]) null).byteLength()); 20 | } 21 | 22 | @Test 23 | public void emptyStringConstructorNullTest() { 24 | assertArrayEquals(new byte[0], new CsvValue("").getBytes()); 25 | assertArrayEquals(new byte[0], new CsvValue("".getBytes(StandardCharsets.UTF_8)).getBytes()); 26 | } 27 | 28 | @Test 29 | public void nonEmptyStringConstructorNullTest() { 30 | assertArrayEquals("42".getBytes(StandardCharsets.UTF_8), new CsvValue("42").getBytes()); 31 | assertArrayEquals("42".getBytes(StandardCharsets.UTF_8), new CsvValue("42".getBytes(StandardCharsets.UTF_8)).getBytes()); 32 | } 33 | 34 | @Test 35 | public void encodeTest() { 36 | assertNull(ValueConverter.String.decode(new CsvValue((String) null).getEncodedBytes())); 37 | assertEquals("", ValueConverter.String.decode(new CsvValue("").getEncodedBytes())); 38 | assertEquals("hello world!", ValueConverter.String.decode(new CsvValue("hello world!").getEncodedBytes())); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/data/UnitsTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.data; 5 | 6 | import org.junit.jupiter.api.Test; 7 | import org.junit.jupiter.params.ParameterizedTest; 8 | import org.junit.jupiter.params.provider.Arguments; 9 | import org.junit.jupiter.params.provider.MethodSource; 10 | 11 | import java.math.BigInteger; 12 | import java.util.stream.Stream; 13 | 14 | import static org.junit.jupiter.api.Assertions.assertEquals; 15 | import static org.junit.jupiter.api.Assertions.assertNull; 16 | 17 | public class UnitsTest { 18 | private static Stream secondConversions() { 19 | return Stream.of( 20 | Arguments.of(2L, 2L, Units.Seconds.MILLIS, Units.Seconds.MILLIS), 21 | Arguments.of(2L, 2000L, Units.Seconds.MILLIS, Units.Seconds.MICROS), 22 | Arguments.of(2L, 2000000L, Units.Seconds.MILLIS, Units.Seconds.NANOS), 23 | Arguments.of(3000L, 3L, Units.Seconds.MICROS, Units.Seconds.MILLIS), 24 | Arguments.of(3L, 3L, Units.Seconds.MICROS, Units.Seconds.MICROS), 25 | Arguments.of(3L, 3000L, Units.Seconds.MICROS, Units.Seconds.NANOS), 26 | Arguments.of(4000000L, 4L, Units.Seconds.NANOS, Units.Seconds.MILLIS), 27 | Arguments.of(4000L, 4L, Units.Seconds.NANOS, Units.Seconds.MICROS), 28 | Arguments.of(4L, 4L, Units.Seconds.NANOS, Units.Seconds.NANOS) 29 | ); 30 | } 31 | 32 | @ParameterizedTest 33 | @MethodSource("secondConversions") 34 | public void secondConversionTest(final Long value, final Long result, final Units.Seconds from, final Units.Seconds to) { 35 | assertEquals(BigInteger.valueOf(result), Units.Seconds.convert(BigInteger.valueOf(value), from, to)); 36 | } 37 | 38 | @Test 39 | public void nullValueSecondConversionTest() { 40 | assertNull(Units.Seconds.convert(null, Units.Seconds.MICROS, Units.Seconds.MICROS)); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/data/ValueTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.data; 5 | 6 | import org.junit.jupiter.api.Test; 7 | 8 | import static org.junit.jupiter.api.Assertions.assertArrayEquals; 9 | import static org.junit.jupiter.api.Assertions.assertEquals; 10 | import static org.junit.jupiter.api.Assertions.assertNull; 11 | import static org.mockito.Mockito.mock; 12 | import static org.mockito.Mockito.when; 13 | 14 | public class ValueTest { 15 | @Test 16 | public void getBytesReturnsNullWhenInputIsNull() { 17 | assertNull(Value.getBytes(null)); 18 | } 19 | 20 | @Test 21 | public void getBytesReturnsNullWhenDataIsNull() { 22 | final Value value = mock(Value.class); 23 | when(value.getBytes()).thenReturn(null); 24 | assertNull(value.getBytes()); 25 | } 26 | 27 | @Test public void getBytesReturnsArray() { 28 | final byte[] bytes = {1, 2, 3, 4}; 29 | final Value value = mock(Value.class); 30 | when(value.getBytes()).thenReturn(bytes); 31 | // Check addresses are the same 32 | assertEquals(bytes, value.getBytes()); 33 | // Check the values are the same 34 | assertArrayEquals(bytes, value.getBytes()); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/encryption/EncryptionContextTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.encryption; 5 | 6 | import com.amazonaws.c3r.config.PadType; 7 | import com.amazonaws.c3r.exception.C3rIllegalArgumentException; 8 | import com.amazonaws.c3r.internal.Nonce; 9 | import org.junit.jupiter.api.Test; 10 | 11 | import java.nio.charset.StandardCharsets; 12 | 13 | import static org.junit.jupiter.api.Assertions.assertArrayEquals; 14 | import static org.junit.jupiter.api.Assertions.assertEquals; 15 | import static org.junit.jupiter.api.Assertions.assertThrows; 16 | 17 | public class EncryptionContextTest { 18 | private final Nonce nonce = new Nonce("nonce01234567890nonce01234567890".getBytes(StandardCharsets.UTF_8)); 19 | 20 | @Test 21 | public void constructorTest() { 22 | final String label = "label"; 23 | final int padLength = 1; 24 | final int maxValueLength = 5; 25 | final EncryptionContext context = EncryptionContext.builder() 26 | .padLength(padLength) 27 | .columnLabel(label) 28 | .padType(PadType.FIXED) 29 | .nonce(nonce) 30 | .maxValueLength(maxValueLength) 31 | .build(); 32 | assertEquals(label, context.getColumnLabel()); 33 | assertEquals(padLength, context.getPadLength()); 34 | assertEquals(maxValueLength, context.getMaxValueLength()); 35 | assertEquals(PadType.FIXED, context.getPadType()); 36 | assertArrayEquals(nonce.getBytes(), context.getNonce().getBytes()); 37 | } 38 | 39 | @Test 40 | public void constructorNullLabelTest() { 41 | final int padLength = 1; 42 | assertThrows(C3rIllegalArgumentException.class, () -> EncryptionContext.builder() 43 | .padLength(padLength) 44 | .columnLabel(null) 45 | .nonce(nonce) 46 | .build()); 47 | } 48 | 49 | @Test 50 | public void constructorEmptyLabelTest() { 51 | final int padLength = 1; 52 | assertThrows(C3rIllegalArgumentException.class, () -> EncryptionContext.builder() 53 | .padLength(padLength) 54 | .columnLabel("") 55 | .nonce(nonce) 56 | .build()); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/encryption/config/ClientSettingsTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.encryption.config; 5 | 6 | import com.amazonaws.c3r.config.ClientSettings; 7 | import org.junit.jupiter.api.Test; 8 | 9 | import static org.junit.jupiter.api.Assertions.assertFalse; 10 | import static org.junit.jupiter.api.Assertions.assertTrue; 11 | 12 | public class ClientSettingsTest { 13 | @Test 14 | public void builderTest() { 15 | final ClientSettings settings = ClientSettings.builder() 16 | .allowDuplicates(true) 17 | .allowJoinsOnColumnsWithDifferentNames(true) 18 | .allowCleartext(true) 19 | .preserveNulls(true).build(); 20 | assertTrue(settings.isAllowDuplicates()); 21 | assertTrue(settings.isAllowJoinsOnColumnsWithDifferentNames()); 22 | assertTrue(settings.isAllowCleartext()); 23 | assertTrue(settings.isPreserveNulls()); 24 | } 25 | 26 | @Test 27 | public void highAssuranceModeTest() { 28 | final ClientSettings settings = ClientSettings.highAssuranceMode(); 29 | assertFalse(settings.isAllowDuplicates()); 30 | assertFalse(settings.isAllowJoinsOnColumnsWithDifferentNames()); 31 | assertFalse(settings.isAllowCleartext()); 32 | assertFalse(settings.isPreserveNulls()); 33 | } 34 | 35 | @Test 36 | public void lowAssuranceModeTest() { 37 | final ClientSettings settings = ClientSettings.lowAssuranceMode(); 38 | assertTrue(settings.isAllowDuplicates()); 39 | assertTrue(settings.isAllowJoinsOnColumnsWithDifferentNames()); 40 | assertTrue(settings.isAllowCleartext()); 41 | assertTrue(settings.isPreserveNulls()); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/encryption/keys/DerivedRootEncryptionKeyTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.encryption.keys; 5 | 6 | import com.amazonaws.c3r.exception.C3rIllegalArgumentException; 7 | import com.amazonaws.c3r.utils.GeneralTestUtility; 8 | import org.junit.jupiter.api.Test; 9 | 10 | import javax.crypto.SecretKey; 11 | import javax.crypto.spec.SecretKeySpec; 12 | import java.nio.charset.StandardCharsets; 13 | 14 | import static com.amazonaws.c3r.encryption.keys.KeyUtil.SHARED_SECRET_KEY_BYTE_LENGTH; 15 | import static org.junit.jupiter.api.Assertions.assertEquals; 16 | import static org.junit.jupiter.api.Assertions.assertThrows; 17 | 18 | public class DerivedRootEncryptionKeyTest { 19 | private final byte[] salt = "saltybytes".getBytes(StandardCharsets.UTF_8); 20 | 21 | private final SecretKey secretKey = 22 | new SecretKeySpec(GeneralTestUtility.EXAMPLE_KEY_BYTES, 0, SHARED_SECRET_KEY_BYTE_LENGTH, KeyUtil.KEY_ALG); 23 | 24 | @Test 25 | public void deriveRootEncryptionKeyTest() { 26 | final DerivedRootEncryptionKey derivedRootEncryptionKey = new DerivedRootEncryptionKey(secretKey, salt); 27 | 28 | assertEquals(KeyUtil.KEY_ALG, derivedRootEncryptionKey.getAlgorithm()); 29 | assertEquals(SHARED_SECRET_KEY_BYTE_LENGTH, derivedRootEncryptionKey.getEncoded().length); 30 | } 31 | 32 | @Test 33 | public void deriveRootEncryptionKeyNullKeyTest() { 34 | assertThrows(C3rIllegalArgumentException.class, () -> new DerivedRootEncryptionKey(null, salt)); 35 | } 36 | 37 | @Test 38 | public void deriveRootEncryptionKeyNullSaltTest() { 39 | assertThrows(C3rIllegalArgumentException.class, () -> new DerivedRootEncryptionKey(secretKey, null)); 40 | } 41 | 42 | @Test 43 | public void deriveRootEncryptionKeyEmptySaltTest() { 44 | assertThrows(C3rIllegalArgumentException.class, () -> new DerivedRootEncryptionKey(secretKey, new byte[0])); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/encryption/keys/KeyTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.encryption.keys; 5 | 6 | import com.amazonaws.c3r.exception.C3rIllegalArgumentException; 7 | import com.amazonaws.c3r.internal.Nonce; 8 | import org.junit.jupiter.api.Test; 9 | 10 | import java.nio.charset.StandardCharsets; 11 | 12 | import static org.junit.jupiter.api.Assertions.assertThrows; 13 | 14 | public class KeyTest { 15 | @Test 16 | public void nullKeyValidationTest() { 17 | final byte[] salt = "saltybytes".getBytes(StandardCharsets.UTF_8); 18 | assertThrows(C3rIllegalArgumentException.class, () -> new DerivedRootEncryptionKey(null, salt)); 19 | assertThrows(C3rIllegalArgumentException.class, () -> new DerivedEncryptionKey(null, Nonce.nextNonce())); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/encryption/keys/SaltedHkdfTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.encryption.keys; 5 | 6 | import com.amazonaws.c3r.exception.C3rIllegalArgumentException; 7 | import com.amazonaws.c3r.utils.GeneralTestUtility; 8 | import org.junit.jupiter.api.Test; 9 | 10 | import javax.crypto.SecretKey; 11 | import javax.crypto.spec.SecretKeySpec; 12 | import java.nio.charset.StandardCharsets; 13 | 14 | import static com.amazonaws.c3r.encryption.keys.KeyUtil.SHARED_SECRET_KEY_BYTE_LENGTH; 15 | import static org.junit.jupiter.api.Assertions.assertThrows; 16 | 17 | public class SaltedHkdfTest { 18 | private final byte[] salt = "saltybytes".getBytes(StandardCharsets.UTF_8); 19 | 20 | private final SecretKey secretKey = 21 | new SecretKeySpec(GeneralTestUtility.EXAMPLE_KEY_BYTES, 0, SHARED_SECRET_KEY_BYTE_LENGTH, KeyUtil.KEY_ALG); 22 | 23 | @Test 24 | public void saltedHkdfNullKeyTest() { 25 | assertThrows(C3rIllegalArgumentException.class, () -> new SaltedHkdf(null, salt)); 26 | } 27 | 28 | @Test 29 | public void saltedHkdfNullSaltTest() { 30 | assertThrows(C3rIllegalArgumentException.class, () -> new SaltedHkdf(secretKey, null)); 31 | } 32 | 33 | @Test 34 | public void saltedHkdfEmptySaltTest() { 35 | assertThrows(C3rIllegalArgumentException.class, () -> new SaltedHkdf(secretKey, new byte[0])); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/encryption/materials/SymmetricRawMaterialsTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.encryption.materials; 5 | 6 | import com.amazonaws.c3r.encryption.keys.DerivedRootEncryptionKey; 7 | import org.junit.jupiter.api.Test; 8 | 9 | import javax.crypto.spec.SecretKeySpec; 10 | import java.nio.charset.StandardCharsets; 11 | 12 | import static org.junit.jupiter.api.Assertions.assertArrayEquals; 13 | 14 | public class SymmetricRawMaterialsTest { 15 | private final byte[] secret = "SomeFakeSecretKey".getBytes(StandardCharsets.UTF_8); 16 | 17 | private final byte[] salt = "saltybytes".getBytes(StandardCharsets.UTF_8); 18 | 19 | private final DerivedRootEncryptionKey secretKey = 20 | new DerivedRootEncryptionKey(new SecretKeySpec(secret, 0, secret.length, "AES"), salt); 21 | 22 | @Test 23 | public void constructorTest() { 24 | final SymmetricRawMaterials materials = new SymmetricRawMaterials(secretKey); 25 | final byte[] encryptionKey = materials.getRootEncryptionKey().getEncoded(); 26 | final byte[] decryptionKey = materials.getRootDecryptionKey().getEncoded(); 27 | assertArrayEquals(encryptionKey, decryptionKey); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/encryption/providers/SymmetricStaticProviderTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.encryption.providers; 5 | 6 | import org.junit.jupiter.api.Test; 7 | 8 | import javax.crypto.SecretKey; 9 | import javax.crypto.spec.SecretKeySpec; 10 | import java.nio.charset.StandardCharsets; 11 | 12 | import static org.junit.jupiter.api.Assertions.assertArrayEquals; 13 | 14 | public class SymmetricStaticProviderTest { 15 | private final byte[] secret = "SomeFakeSecretKey".getBytes(StandardCharsets.UTF_8); 16 | 17 | private final SecretKey secretKey = new SecretKeySpec(secret, 0, secret.length, "AES"); 18 | 19 | private final byte[] salt = "saltybytes".getBytes(StandardCharsets.UTF_8); 20 | 21 | @Test 22 | public void constructorTest() { 23 | final SymmetricStaticProvider provider = new SymmetricStaticProvider(secretKey, salt); 24 | final byte[] encryptionKey = provider.getEncryptionMaterials(null).getRootEncryptionKey().getEncoded(); 25 | final byte[] decryptionKey = provider.getDecryptionMaterials(null).getRootDecryptionKey().getEncoded(); 26 | assertArrayEquals(encryptionKey, decryptionKey); 27 | } 28 | 29 | @Test 30 | public void refreshTest() { 31 | final SymmetricStaticProvider provider = new SymmetricStaticProvider(secretKey, salt); 32 | final byte[] encryptionKey = provider.getEncryptionMaterials(null).getRootEncryptionKey().getEncoded(); 33 | final byte[] decryptionKey = provider.getDecryptionMaterials(null).getRootDecryptionKey().getEncoded(); 34 | 35 | provider.refresh(); 36 | final byte[] refreshedEncryptionKey = provider.getEncryptionMaterials(null).getRootEncryptionKey().getEncoded(); 37 | final byte[] refreshedDecryptionKey = provider.getDecryptionMaterials(null).getRootDecryptionKey().getEncoded(); 38 | assertArrayEquals(encryptionKey, refreshedEncryptionKey); 39 | assertArrayEquals(decryptionKey, refreshedDecryptionKey); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/exception/C3rIllegalArgumentExceptionTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.exception; 5 | 6 | import org.junit.jupiter.api.Test; 7 | 8 | import static org.junit.jupiter.api.Assertions.assertEquals; 9 | 10 | public class C3rIllegalArgumentExceptionTest { 11 | private final Exception sensitiveException = new RuntimeException("sensitive message"); 12 | 13 | @Test 14 | public void getMessageTest() { 15 | // Ensure just the given message is returned and never an underlying message 16 | assertEquals("Doh!", new C3rIllegalArgumentException("Doh!").getMessage()); 17 | assertEquals("Doh!", 18 | new C3rIllegalArgumentException("Doh!", sensitiveException).getMessage()); 19 | } 20 | 21 | @Test 22 | public void getCauseTest() { 23 | // verify the underlying cause exception is being stored and returned as expected 24 | assertEquals(sensitiveException, new C3rIllegalArgumentException("doh!", sensitiveException).getCause()); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/exception/C3rRuntimeExceptionTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.exception; 5 | 6 | import org.junit.jupiter.api.Test; 7 | 8 | import static org.junit.jupiter.api.Assertions.assertEquals; 9 | 10 | public class C3rRuntimeExceptionTest { 11 | private final Exception sensitiveException = new RuntimeException("sensitive message"); 12 | 13 | @Test 14 | public void getMessageTest() { 15 | // Ensure just the given message is returned and never an underlying message 16 | assertEquals("Doh!", new C3rRuntimeException("Doh!").getMessage()); 17 | assertEquals("Doh!", 18 | new C3rRuntimeException("Doh!", sensitiveException).getMessage()); 19 | } 20 | 21 | @Test 22 | public void getCauseTest() { 23 | // verify the underlying cause exception is being stored and returned as expected 24 | assertEquals(sensitiveException, new C3rRuntimeException("doh!", sensitiveException).getCause()); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/internal/ColumnTypeTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.internal; 5 | 6 | import com.amazonaws.c3r.CleartextTransformer; 7 | import com.amazonaws.c3r.FingerprintTransformer; 8 | import com.amazonaws.c3r.SealedTransformer; 9 | import com.amazonaws.c3r.config.ColumnType; 10 | import org.junit.jupiter.api.Test; 11 | 12 | import static org.junit.jupiter.api.Assertions.assertEquals; 13 | 14 | public class ColumnTypeTest { 15 | 16 | @Test 17 | public void cleartextColumnTypeTest() { 18 | assertEquals(CleartextTransformer.class, ColumnType.CLEARTEXT.getTransformerType()); 19 | } 20 | 21 | @Test 22 | public void fingerprintColumnTypeTest() { 23 | assertEquals(FingerprintTransformer.class, ColumnType.FINGERPRINT.getTransformerType()); 24 | } 25 | 26 | @Test 27 | public void sealedColumnTypeTest() { 28 | assertEquals(SealedTransformer.class, ColumnType.SEALED.getTransformerType()); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/internal/NonceTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.internal; 5 | 6 | import com.amazonaws.c3r.exception.C3rIllegalArgumentException; 7 | import org.junit.jupiter.api.RepeatedTest; 8 | import org.junit.jupiter.api.Test; 9 | 10 | import java.nio.charset.StandardCharsets; 11 | import java.util.Arrays; 12 | 13 | import static org.junit.jupiter.api.Assertions.assertFalse; 14 | import static org.junit.jupiter.api.Assertions.assertThrows; 15 | 16 | public class NonceTest { 17 | 18 | @RepeatedTest(100) 19 | public void getNonceTest() { 20 | // Ensure nonces aren't equal 21 | final Nonce nonce1 = Nonce.nextNonce(); 22 | final Nonce nonce2 = Nonce.nextNonce(); 23 | assertFalse(Arrays.equals(nonce1.getBytes(), nonce2.getBytes())); 24 | } 25 | 26 | @Test 27 | public void validateNullNonceTest() { 28 | assertThrows(C3rIllegalArgumentException.class, () -> new Nonce(null)); 29 | } 30 | 31 | @Test 32 | public void validateEmptyNonceTest() { 33 | assertThrows(C3rIllegalArgumentException.class, () -> new Nonce(new byte[0])); 34 | } 35 | 36 | @Test 37 | public void validateSmallNonceTest() { 38 | final byte[] nonceBytes = "small".getBytes(StandardCharsets.UTF_8); 39 | assertThrows(C3rIllegalArgumentException.class, () -> new Nonce(nonceBytes)); 40 | } 41 | 42 | @Test 43 | public void validateLargeNonceTest() { 44 | final byte[] nonceBytes = ("ANonceMayOnlyBe" + Nonce.NONCE_BYTE_LENGTH + "BytesLong").getBytes(StandardCharsets.UTF_8); 45 | assertThrows(C3rIllegalArgumentException.class, () -> new Nonce(nonceBytes)); 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/io/FileFormatTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.io; 5 | 6 | import org.junit.jupiter.api.Test; 7 | 8 | import static org.junit.jupiter.api.Assertions.assertEquals; 9 | import static org.junit.jupiter.api.Assertions.assertNull; 10 | 11 | public class FileFormatTest { 12 | @Test 13 | public void fromFileNameCsvTest() { 14 | assertEquals(FileFormat.CSV, FileFormat.fromFileName("hello.csv")); 15 | } 16 | 17 | @Test 18 | public void fromFileNameParquetTest() { 19 | assertEquals(FileFormat.PARQUET, FileFormat.fromFileName("hello.parquet")); 20 | } 21 | 22 | @Test 23 | public void fromFileNameUnknownTest() { 24 | assertNull(FileFormat.fromFileName("hello.unknown")); 25 | } 26 | 27 | @Test 28 | public void fromFileNameEmptyTest() { 29 | assertNull(FileFormat.fromFileName("")); 30 | } 31 | 32 | @Test 33 | public void fromFileNameNoSuffixTest() { 34 | assertNull(FileFormat.fromFileName("hello")); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/io/RowReaderTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.io; 5 | 6 | import com.amazonaws.c3r.data.CsvRow; 7 | import com.amazonaws.c3r.data.CsvValue; 8 | import com.amazonaws.c3r.exception.C3rRuntimeException; 9 | import com.amazonaws.c3r.internal.Limits; 10 | import org.junit.jupiter.api.Test; 11 | import org.mockito.Mockito; 12 | 13 | import static org.junit.jupiter.api.Assertions.assertThrows; 14 | import static org.mockito.Mockito.mock; 15 | import static org.mockito.Mockito.when; 16 | 17 | public class RowReaderTest { 18 | @Test 19 | public void setTooManyRecordsInATableTest() { 20 | @SuppressWarnings("unchecked") 21 | final RowReader reader = mock(RowReader.class, Mockito.CALLS_REAL_METHODS); 22 | when(reader.peekNextRow()).thenReturn(new CsvRow()); 23 | assertThrows(C3rRuntimeException.class, () -> reader.setReadRowCount(Limits.ROW_COUNT_MAX + 1)); 24 | } 25 | 26 | @Test 27 | public void readTooManyRecordsInATableTest() { 28 | @SuppressWarnings("unchecked") 29 | final RowReader reader = mock(RowReader.class, Mockito.CALLS_REAL_METHODS); 30 | when(reader.peekNextRow()).thenReturn(new CsvRow()); 31 | reader.setReadRowCount(Limits.ROW_COUNT_MAX); 32 | assertThrows(C3rRuntimeException.class, reader::next); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/json/ColumnTypeTypeAdapterTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.json; 5 | 6 | import com.amazonaws.c3r.config.ColumnType; 7 | import com.amazonaws.c3r.exception.C3rIllegalArgumentException; 8 | import org.junit.jupiter.api.Test; 9 | 10 | import static org.junit.jupiter.api.Assertions.assertEquals; 11 | import static org.junit.jupiter.api.Assertions.assertNull; 12 | import static org.junit.jupiter.api.Assertions.assertThrows; 13 | 14 | public class ColumnTypeTypeAdapterTest { 15 | @Test 16 | public void fromLowerCaseTest() { 17 | final ColumnType columnType = GsonUtil.fromJson("sealed", ColumnType.class); 18 | assertEquals(ColumnType.SEALED, columnType); 19 | } 20 | 21 | @Test 22 | public void fromUpperCaseTest() { 23 | final ColumnType columnType = GsonUtil.fromJson("SEALED", ColumnType.class); 24 | assertEquals(ColumnType.SEALED, columnType); 25 | } 26 | 27 | @Test 28 | public void fromMixedCaseTest() { 29 | final ColumnType columnType = GsonUtil.fromJson("SeaLeD", ColumnType.class); 30 | assertEquals(ColumnType.SEALED, columnType); 31 | } 32 | 33 | @Test 34 | public void badColumnTypeTest() { 35 | assertThrows(C3rIllegalArgumentException.class, () -> GsonUtil.fromJson("bad", ColumnType.class)); 36 | } 37 | 38 | @Test 39 | public void readNullTest() { 40 | assertNull(GsonUtil.fromJson("null", ColumnType.class)); 41 | } 42 | 43 | @Test 44 | public void writeTest() { 45 | final String serialized = GsonUtil.toJson(ColumnType.SEALED); 46 | final String expected = "\"" + ColumnType.SEALED + "\""; 47 | assertEquals(expected, serialized); 48 | } 49 | 50 | @Test 51 | public void writeNullTest() { 52 | final String serialized = GsonUtil.toJson(null); 53 | assertEquals("null", serialized); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/json/PadTypeTypeAdapterTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.json; 5 | 6 | import com.amazonaws.c3r.config.PadType; 7 | import com.amazonaws.c3r.exception.C3rIllegalArgumentException; 8 | import org.junit.jupiter.api.Test; 9 | 10 | import static org.junit.jupiter.api.Assertions.assertEquals; 11 | import static org.junit.jupiter.api.Assertions.assertNull; 12 | import static org.junit.jupiter.api.Assertions.assertThrows; 13 | 14 | public class PadTypeTypeAdapterTest { 15 | @Test 16 | public void fromLowerCaseTest() { 17 | final PadType padType = GsonUtil.fromJson("max", PadType.class); 18 | assertEquals(PadType.MAX, padType); 19 | } 20 | 21 | @Test 22 | public void fromUpperCaseTest() { 23 | final PadType padType = GsonUtil.fromJson("MAX", PadType.class); 24 | assertEquals(PadType.MAX, padType); 25 | } 26 | 27 | @Test 28 | public void fromMixedCaseTest() { 29 | final PadType padType = GsonUtil.fromJson("MaX", PadType.class); 30 | assertEquals(PadType.MAX, padType); 31 | } 32 | 33 | @Test 34 | public void badPadTypeTest() { 35 | assertThrows(C3rIllegalArgumentException.class, () -> GsonUtil.fromJson("bad", PadType.class)); 36 | } 37 | 38 | @Test 39 | public void readNullTest() { 40 | assertNull(GsonUtil.fromJson("null", PadType.class)); 41 | } 42 | 43 | @Test 44 | public void writeTest() { 45 | final String serialized = GsonUtil.toJson(PadType.MAX); 46 | final String expected = "\"" + PadType.MAX.name() + "\""; 47 | assertEquals(expected, serialized); 48 | } 49 | 50 | @Test 51 | public void writeNullTest() { 52 | final String serialized = GsonUtil.toJson(null); 53 | assertEquals("null", serialized); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/utils/C3rSdkPropertiesTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.utils; 5 | 6 | import org.junit.jupiter.api.Test; 7 | 8 | import java.io.IOException; 9 | import java.nio.charset.StandardCharsets; 10 | import java.nio.file.Files; 11 | import java.nio.file.Path; 12 | 13 | import static org.junit.jupiter.api.Assertions.assertEquals; 14 | 15 | public class C3rSdkPropertiesTest { 16 | 17 | @Test 18 | public void apiNameVersionTest() { 19 | assertEquals(C3rSdkProperties.VERSION, C3rSdkProperties.API_NAME.version()); 20 | } 21 | 22 | @Test 23 | public void apiNameNameTest() { 24 | assertEquals("c3r-sdk", C3rSdkProperties.API_NAME.name()); 25 | } 26 | 27 | @Test 28 | public void externalAndInternalVersionsMatchTest() throws IOException { 29 | // Ensure the version in `version.txt` (used by the build system and any other tooling outside the code base) 30 | // and the version constant in our codebase match. 31 | final String version = Files.readString(Path.of("../version.txt"), StandardCharsets.UTF_8).trim(); 32 | assertEquals(version, C3rSdkProperties.VERSION); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/utils/DecryptSdkConfigTestUtility.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.utils; 5 | 6 | import lombok.Builder; 7 | import lombok.Getter; 8 | 9 | import javax.crypto.spec.SecretKeySpec; 10 | 11 | /** 12 | * Basic Decryption settings. 13 | */ 14 | @Builder 15 | @Getter 16 | public class DecryptSdkConfigTestUtility { 17 | /** 18 | * Key to use for decryption. 19 | */ 20 | @Builder.Default 21 | private SecretKeySpec key = null; 22 | 23 | /** 24 | * Salt for key generation. 25 | */ 26 | @Builder.Default 27 | private String salt = null; 28 | 29 | /** 30 | * Input file. 31 | */ 32 | @Builder.Default 33 | private String input = null; 34 | 35 | /** 36 | * Column header names. 37 | */ 38 | @Builder.Default 39 | private String[] columnHeaders = null; 40 | } 41 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/java/com/amazonaws/c3r/utils/EncryptSdkConfigTestUtility.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.utils; 5 | 6 | import com.amazonaws.c3r.config.ClientSettings; 7 | import com.amazonaws.c3r.config.TableSchema; 8 | import lombok.Builder; 9 | import lombok.Getter; 10 | 11 | import javax.crypto.spec.SecretKeySpec; 12 | import java.util.List; 13 | 14 | /** 15 | * Basic configuration settings for encryption. 16 | */ 17 | @Builder 18 | @Getter 19 | public class EncryptSdkConfigTestUtility { 20 | /** 21 | * Schema specification. 22 | */ 23 | @Builder.Default 24 | private TableSchema schema = null; 25 | 26 | /** 27 | * Key to use for encryption. 28 | */ 29 | @Builder.Default 30 | private SecretKeySpec key = null; 31 | 32 | /** 33 | * Salt to use for key generation. 34 | */ 35 | @Builder.Default 36 | private String salt = null; 37 | 38 | /** 39 | * Security related parameters. 40 | */ 41 | @Builder.Default 42 | private ClientSettings settings = ClientSettings.lowAssuranceMode(); 43 | 44 | /** 45 | * Input file. 46 | */ 47 | @Builder.Default 48 | private String input = null; 49 | 50 | /** 51 | * Column headers in the input file. 52 | */ 53 | @Builder.Default 54 | private List inputColumnHeaders = null; 55 | 56 | /** 57 | * Column headers to use in the output file. 58 | */ 59 | @Builder.Default 60 | private List outputColumnHeaders = null; 61 | } 62 | -------------------------------------------------------------------------------- /c3r-sdk-core/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | logs 5 | 6 | 7 | 9 | 10 | %d{dd MMM yyyy HH:mm:ss,SSS} [%p] (%t) %c: %m%n%ex 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | %d{dd MMM yyyy HH:mm:ss,SSS} [%p] (%t) %c: %m%n%ex 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /c3r-sdk-examples/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | // Java compilation, unit tests, and library distribution needs. Read more at: 3 | // https://docs.gradle.org/current/userguide/java_library_plugin.html 4 | id 'java-library' 5 | 6 | // SpotBugs for quality checks and reports of source files. Read more at: 7 | // https://spotbugs.readthedocs.io/en/stable/gradle.html 8 | id "com.github.spotbugs" version "$spotbugs_version" 9 | } 10 | 11 | apply plugin: 'com.github.spotbugs' 12 | 13 | /* 14 | Configures the SpotBugs "com.github.spotbugs" plugin. Remove this and the 15 | plugin to skip these checks and report generation. 16 | */ 17 | spotbugs { 18 | ignoreFailures.set(false) 19 | spotbugsTest.enabled = false 20 | excludeFilter.set(file("../config/spotbugs/excludeFilter.xml")) 21 | } 22 | 23 | dependencies { 24 | // AWS-C3R 25 | implementation project(":c3r-sdk-core") 26 | implementation project(":c3r-sdk-parquet") 27 | 28 | // Spark 29 | implementation "org.apache.spark:spark-core_2.13:$spark_version" 30 | implementation "org.apache.spark:spark-streaming_2.13:$spark_version" 31 | implementation "org.apache.spark:spark-sql_2.13:$spark_version" 32 | } 33 | 34 | test { 35 | useJUnitPlatform() 36 | 37 | // Always run tests, even when nothing changed. 38 | dependsOn 'cleanTest' 39 | 40 | // Show test results. 41 | testLogging { 42 | events "failed" 43 | 44 | showExceptions true 45 | exceptionFormat "full" 46 | showCauses true 47 | showStackTraces true 48 | 49 | showStandardStreams = false 50 | } 51 | 52 | // Required for Spark. While Spark supports Java 17, it still makes a call to this class 53 | // and will cause a runtime failure. 54 | jvmArgs '--add-exports=java.base/sun.nio.ch=ALL-UNNAMED' 55 | } 56 | -------------------------------------------------------------------------------- /c3r-sdk-examples/src/main/java/com/amazonaws/c3r/examples/csv/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Sample code showing how to use the SDK with CSV data. 6 | * 7 | *

8 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 9 | * SPDX-License-Identifier: Apache-2.0 10 | */ 11 | package com.amazonaws.c3r.examples.csv; -------------------------------------------------------------------------------- /c3r-sdk-examples/src/main/java/com/amazonaws/c3r/examples/parquet/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Sample code showing how to use the SDK with Parquet data. 6 | * 7 | *

8 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 9 | * SPDX-License-Identifier: Apache-2.0 10 | */ 11 | package com.amazonaws.c3r.examples.parquet; -------------------------------------------------------------------------------- /c3r-sdk-examples/src/main/java/com/amazonaws/c3r/examples/spark/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Sample code showing how to use the SDK with Spark. 6 | * 7 | *

8 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 9 | * SPDX-License-Identifier: Apache-2.0 10 | */ 11 | package com.amazonaws.c3r.examples.spark; -------------------------------------------------------------------------------- /c3r-sdk-examples/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 9 | logs 10 | 11 | 12 | 14 | 15 | %d{dd MMM yyyy HH:mm:ss,SSS} [%p] (%t) %c: %m%n%ex 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | %d{dd MMM yyyy HH:mm:ss,SSS} [%p] (%t) %c: %m%n%ex 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /c3r-sdk-examples/src/test/java/com/amazonaws/c3r/examples/csv/CsvExampleTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.examples.csv; 5 | 6 | import com.amazonaws.c3r.examples.utils.FileTestUtility; 7 | import org.junit.jupiter.api.Test; 8 | 9 | import java.io.IOException; 10 | import java.nio.file.Files; 11 | import java.nio.file.Path; 12 | 13 | import static org.junit.jupiter.api.Assertions.assertTrue; 14 | 15 | public class CsvExampleTest { 16 | @Test 17 | public void roundTripTest() throws IOException { 18 | final Path inputCsv = Path.of("../samples/csv/data_sample_without_quotes.csv"); 19 | final Path encryptedCsv = FileTestUtility.createTempFile("encrypted", ".csv"); 20 | final Path decryptedCsv = FileTestUtility.createTempFile("decrypted", ".csv"); 21 | 22 | CsvExample.encrypt(inputCsv.toString(), encryptedCsv.toString()); 23 | assertTrue(Files.size(encryptedCsv) > 0); 24 | 25 | CsvExample.decrypt(encryptedCsv.toString(), decryptedCsv.toString()); 26 | 27 | assertTrue(Files.size(decryptedCsv) > 0); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /c3r-sdk-examples/src/test/java/com/amazonaws/c3r/examples/csv/CsvNoHeaderExampleTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.examples.csv; 5 | 6 | import com.amazonaws.c3r.examples.utils.FileTestUtility; 7 | import org.junit.jupiter.api.Test; 8 | 9 | import java.io.IOException; 10 | import java.nio.file.Files; 11 | import java.nio.file.Path; 12 | 13 | import static org.junit.jupiter.api.Assertions.assertTrue; 14 | 15 | public class CsvNoHeaderExampleTest { 16 | @Test 17 | public void roundTripTest() throws IOException { 18 | final Path inputCsv = Path.of("../samples/csv/data_sample_no_headers.csv"); 19 | final Path encryptedCsv = FileTestUtility.createTempFile("encrypted", ".csv"); 20 | final Path decryptedCsv = FileTestUtility.createTempFile("decrypted", ".csv"); 21 | 22 | CsvNoHeaderExample.encrypt(inputCsv.toString(), encryptedCsv.toString()); 23 | assertTrue(Files.size(encryptedCsv) > 0); 24 | 25 | CsvNoHeaderExample.decrypt(encryptedCsv.toString(), decryptedCsv.toString()); 26 | 27 | assertTrue(Files.size(decryptedCsv) > 0); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /c3r-sdk-examples/src/test/java/com/amazonaws/c3r/examples/parquet/ParquetExampleTest.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.examples.parquet; 5 | 6 | import com.amazonaws.c3r.examples.utils.FileTestUtility; 7 | import org.junit.jupiter.api.Test; 8 | 9 | import java.io.IOException; 10 | import java.nio.file.Files; 11 | import java.nio.file.Path; 12 | 13 | import static org.junit.jupiter.api.Assertions.assertTrue; 14 | 15 | public class ParquetExampleTest { 16 | 17 | @Test 18 | public void roundTripTest() throws IOException { 19 | final Path inputParquet = Path.of("../samples/parquet/data_sample.parquet"); 20 | final Path encryptedParquet = FileTestUtility.createTempFile("encrypted", ".parquet"); 21 | final Path decryptedParquet = FileTestUtility.createTempFile("decrypted", ".parquet"); 22 | 23 | ParquetExample.encrypt(inputParquet.toString(), encryptedParquet.toString()); 24 | assertTrue(Files.size(encryptedParquet) > 0); 25 | 26 | ParquetExample.decrypt(encryptedParquet.toString(), decryptedParquet.toString()); 27 | 28 | assertTrue(Files.size(decryptedParquet) > 0); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /c3r-sdk-examples/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 9 | logs 10 | 11 | 12 | 14 | 15 | %d{dd MMM yyyy HH:mm:ss,SSS} [%p] (%t) %c: %m%n%ex 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | %d{dd MMM yyyy HH:mm:ss,SSS} [%p] (%t) %c: %m%n%ex 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /c3r-sdk-parquet/src/main/java/com/amazonaws/c3r/action/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * This package classes used to marshall (encrypt) and unmarshall (decrypt) data to and from the clean room for Parquet files. 6 | * {@link com.amazonaws.c3r.action.RowMarshaller} handles the logic of marshalling data outside of anything having to do with 7 | * Parquet itself and {@link com.amazonaws.c3r.action.RowUnmarshaller} does the same for unmarshalling. 8 | * 9 | *

10 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 11 | * SPDX-License-Identifier: Apache-2.0 12 | */ 13 | package com.amazonaws.c3r.action; -------------------------------------------------------------------------------- /c3r-sdk-parquet/src/main/java/com/amazonaws/c3r/config/ParquetConfig.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.config; 5 | 6 | import lombok.Builder; 7 | import lombok.Value; 8 | 9 | /** 10 | * Information needed for processing a Parquet files. 11 | */ 12 | @Value 13 | @Builder 14 | public class ParquetConfig { 15 | /** 16 | * An instance of ParquetConfig using only default values. 17 | */ 18 | public static final ParquetConfig DEFAULT = new ParquetConfig(false); 19 | 20 | /** 21 | * Treat Parquet Binary values without annotations as if they had the string annotation. 22 | */ 23 | private final Boolean binaryAsString; 24 | 25 | /** 26 | * Checks if any of the Parquet settings are configured. Used for validating that specified CLI options match file type. 27 | * 28 | * @return {@code true} If any configuration option is set. 29 | */ 30 | public boolean isSet() { 31 | return binaryAsString != null; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /c3r-sdk-parquet/src/main/java/com/amazonaws/c3r/config/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Classes that contain extra information needed to perform cryptographic computations on Parquet data. 6 | * 7 | *

8 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 9 | * SPDX-License-Identifier: Apache-2.0 10 | */ 11 | package com.amazonaws.c3r.config; -------------------------------------------------------------------------------- /c3r-sdk-parquet/src/main/java/com/amazonaws/c3r/data/ParquetRow.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.data; 5 | 6 | import com.amazonaws.c3r.config.ColumnHeader; 7 | import com.amazonaws.c3r.internal.Nonce; 8 | import lombok.NonNull; 9 | 10 | import java.util.Map; 11 | import java.util.Objects; 12 | 13 | /** 14 | * Row of Parquet data. 15 | */ 16 | public final class ParquetRow extends Row { 17 | /** 18 | * Maps column name to data type. 19 | */ 20 | private final Map columnTypes; 21 | 22 | /** 23 | * Stores the Parquet values for a particular row. 24 | * 25 | * @param columnTypes Maps column name to data type in column 26 | */ 27 | public ParquetRow(final Map columnTypes) { 28 | this.columnTypes = Map.copyOf(columnTypes); 29 | } 30 | 31 | /** 32 | * {@inheritDoc} 33 | */ 34 | @Override 35 | public void putBytes(@NonNull final ColumnHeader column, final byte[] bytes) { 36 | final ParquetDataType parquetType = Objects.requireNonNull(this.columnTypes.get(column), 37 | "Internal error! Unknown column: " + column); 38 | putValue(column, ParquetValue.fromBytes(parquetType, bytes)); 39 | } 40 | 41 | /** 42 | * {@inheritDoc} 43 | */ 44 | @Override 45 | public void putNonce(@NonNull final ColumnHeader nonceColumn, final Nonce nonce) { 46 | putValue(nonceColumn, ParquetValue.fromBytes(ParquetDataType.NONCE_TYPE, nonce.getBytes())); 47 | } 48 | 49 | @Override 50 | public Row clone() { 51 | final Row row = new ParquetRow(columnTypes); 52 | getHeaders().forEach(header -> row.putValue(header, getValue(header))); 53 | return row; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /c3r-sdk-parquet/src/main/java/com/amazonaws/c3r/data/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Implementations of {@link com.amazonaws.c3r.data.Value}, {@link com.amazonaws.c3r.data.Row}, and 6 | * {@link com.amazonaws.c3r.data.ValueFactory} for Parquet data. The additional classes to support processing the data, such as schemas and 7 | * type metadata classes are contained here as well. 8 | * 9 | *

10 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 11 | * SPDX-License-Identifier: Apache-2.0 12 | */ 13 | package com.amazonaws.c3r.data; -------------------------------------------------------------------------------- /c3r-sdk-parquet/src/main/java/com/amazonaws/c3r/io/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Support for reading and writing rows of Parquet data. 6 | * 7 | *

8 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 9 | * SPDX-License-Identifier: Apache-2.0 10 | */ 11 | package com.amazonaws.c3r.io; -------------------------------------------------------------------------------- /c3r-sdk-parquet/src/main/java/com/amazonaws/c3r/io/parquet/ParquetRowMaterializer.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package com.amazonaws.c3r.io.parquet; 5 | 6 | import com.amazonaws.c3r.data.ParquetSchema; 7 | import com.amazonaws.c3r.data.ParquetValue; 8 | import com.amazonaws.c3r.data.Row; 9 | import com.amazonaws.c3r.data.ValueFactory; 10 | import org.apache.parquet.io.api.GroupConverter; 11 | import org.apache.parquet.io.api.RecordMaterializer; 12 | 13 | /** 14 | * Takes raw Parquet data and turns it into usable Java values. 15 | */ 16 | public class ParquetRowMaterializer extends RecordMaterializer> { 17 | /** 18 | * Converts raw data into Java values. 19 | */ 20 | private final ParquetRowConverter root; 21 | 22 | /** 23 | * Set up a converter for a given schema specification along with a row generator for Parquet data. 24 | * 25 | * @param schema Description of how data maps to columns, including associated metadata for each type 26 | * @param valueFactory Generate new empty rows to store Parquet data in 27 | */ 28 | public ParquetRowMaterializer(final ParquetSchema schema, final ValueFactory valueFactory) { 29 | // Creates a new row for Parquet values to be stored in 30 | root = new ParquetRowConverter(schema, valueFactory); 31 | } 32 | 33 | /** 34 | * {@inheritDoc} 35 | */ 36 | @Override 37 | public Row getCurrentRecord() { 38 | return root.getRow(); 39 | } 40 | 41 | /** 42 | * Converter for Parquet data into values. 43 | * 44 | * @return Top level converter for transforming Parquet data into Java objects 45 | */ 46 | @Override 47 | public GroupConverter getRootConverter() { 48 | return root; 49 | } 50 | } -------------------------------------------------------------------------------- /c3r-sdk-parquet/src/main/java/com/amazonaws/c3r/io/parquet/package-info.java: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | /** 5 | * Support classes for managing I/O access to Parquet files. Handles the type metadata that needs to be associated with each value, 6 | * as well as conversion to/from the binary format of Parquet to Java values. 7 | * 8 | *

9 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 10 | * SPDX-License-Identifier: Apache-2.0 11 | */ 12 | package com.amazonaws.c3r.io.parquet; -------------------------------------------------------------------------------- /codebuild/release.yml: -------------------------------------------------------------------------------- 1 | version: 0.2 2 | 3 | phases: 4 | install: 5 | runtime-versions: 6 | java: corretto17 7 | commands: 8 | - apt-get install -y python3 9 | - pip3 install --upgrade botocore boto3 10 | build: 11 | commands: 12 | - ./codebuild/build-and-test-release-artifacts.sh unsigned-artifacts 13 | - mkdir signed-artifacts 14 | - python3 codebuild/sign_artifacts.py 15 | artifacts: 16 | files: 17 | - '*.jar' 18 | base-directory: signed-artifacts 19 | -------------------------------------------------------------------------------- /codebuild/release/release-prod.yml: -------------------------------------------------------------------------------- 1 | ## Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | ## SPDX-License-Identifier: Apache-2.0 3 | 4 | version: 0.2 5 | 6 | env: 7 | variables: 8 | BRANCH: "master" 9 | secrets-manager: 10 | GPG_KEY: Maven-GPG-Keys-Release-Credentials:Keyname 11 | GPG_PASS: Maven-GPG-Keys-Release-Credentials:Passphrase 12 | SONA_USERNAME: Sonatype-Team-Account:Username 13 | SONA_PASSWORD: Sonatype-Team-Account:Password 14 | 15 | phases: 16 | install: 17 | runtime-versions: 18 | java: corretto11 19 | pre_build: 20 | commands: 21 | - git checkout $BRANCH 22 | - export SETTINGS_FILE=$(pwd)/codebuild/release/settings.xml 23 | - aws secretsmanager get-secret-value --region us-west-2 --secret-id Maven-GPG-Keys-Release --query SecretBinary --output text | base64 -d > ~/mvn_gpg.tgz 24 | - tar -xvf ~/mvn_gpg.tgz -C ~ 25 | build: 26 | commands: 27 | - | 28 | mvn deploy \ 29 | -Ppublishing \ 30 | -Pfast-tests-only \ 31 | -DperformRelease \ 32 | -Dgpg.homedir="$HOME/mvn_gpg" \ 33 | -DautoReleaseAfterClose=true \ 34 | -Dgpg.keyname="$GPG_KEY" \ 35 | -Dgpg.passphrase="$GPG_PASS" \ 36 | -Dsonatype.username="$SONA_USERNAME" \ 37 | -Dsonatype.password="$SONA_PASSWORD" \ 38 | --no-transfer-progress \ 39 | -s $SETTINGS_FILE -------------------------------------------------------------------------------- /codebuild/release/release-staging.yml: -------------------------------------------------------------------------------- 1 | ## Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | ## SPDX-License-Identifier: Apache-2.0 3 | 4 | version: 0.2 5 | 6 | env: 7 | variables: 8 | REGION: us-east-1 9 | DOMAIN: crypto-compute-internal 10 | REPOSITORY: c3r-staging 11 | parameter-store: 12 | ACCOUNT: /CodeBuild/AccountId 13 | secrets-manager: 14 | GPG_KEY: Maven-GPG-Keys-Release-Credentials:Keyname 15 | GPG_PASS: Maven-GPG-Keys-Release-Credentials:Passphrase 16 | 17 | phases: 18 | install: 19 | runtime-versions: 20 | java: corretto11 21 | pre_build: 22 | commands: 23 | - export SETTINGS_FILE=$(pwd)/codebuild/release/settings.xml 24 | - export CODEARTIFACT_TOKEN=$(aws codeartifact get-authorization-token --domain $DOMAIN --domain-owner $ACCOUNT --query authorizationToken --output text --region ${REGION}) 25 | - export CODEARTIFACT_REPO_URL=https://${DOMAIN}-${ACCOUNT}.d.codeartifact.${REGION}.amazonaws.com/maven/${REPOSITORY} 26 | - aws secretsmanager get-secret-value --region us-west-2 --secret-id Maven-GPG-Keys-Release --query SecretBinary --output text | base64 -d > ~/mvn_gpg.tgz 27 | - tar -xvf ~/mvn_gpg.tgz -C ~ 28 | build: 29 | commands: 30 | - VERSION_HASH="$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)-$CODEBUILD_RESOLVED_SOURCE_VERSION" 31 | # See https://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref-env-vars.html 32 | - echo "Setting version in POM to $VERSION_HASH" 33 | - mvn versions:set -DnewVersion="$VERSION_HASH" --no-transfer-progress 34 | - echo "Version is now $(grep version pom.xml | head -n 1 | sed -n 's/[ \t]*\(.*\)<\/version>/\1/p')" 35 | - | 36 | mvn deploy \ 37 | -PpublishingCodeArtifact \ 38 | -Pfast-tests-only \ 39 | -DperformRelease \ 40 | -Dgpg.homedir="$HOME/mvn_gpg" \ 41 | -DautoReleaseAfterClose=true \ 42 | -Dgpg.keyname="$GPG_KEY" \ 43 | -Dgpg.passphrase="$GPG_PASS" \ 44 | -Dcodeartifact.token=$CODEARTIFACT_TOKEN \ 45 | -DaltDeploymentRepository=codeartifact::default::$CODEARTIFACT_REPO_URL \ 46 | --no-transfer-progress \ 47 | -s $SETTINGS_FILE -------------------------------------------------------------------------------- /codebuild/release/settings.xml: -------------------------------------------------------------------------------- 1 | 5 | 9 | 10 | 11 | codeartifact 12 | aws 13 | ${codeartifact.token} 14 | 15 | 16 | sonatype-nexus-staging 17 | ${sonatype.username} 18 | ${sonatype.password} 19 | 20 | 21 | 22 | 23 | 24 | codeartifact 25 | 26 | 27 | codeartifact 28 | codeartifact 29 | ${codeartifact.url} 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /codebuild/release/upload-artifacts.yml: -------------------------------------------------------------------------------- 1 | ## Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 2 | ## SPDX-License-Identifier: Apache-2.0 3 | 4 | version: 0.2 5 | 6 | env: 7 | variables: 8 | BRANCH: "main" 9 | git-credential-helper: yes 10 | secrets-manager: 11 | GH_TOKEN: Github/crypto-compute-ci-bot:C3R Release Token 12 | 13 | phases: 14 | pre_build: 15 | commands: 16 | # get new project version 17 | - git checkout $BRANCH 18 | - export VERSION=$(grep version pom.xml | head -n 1 | sed -n 's/[ \t]*\(.*\)<\/version>/\1/p') 19 | - git config --global user.name "crypto-compute-ci-bot" 20 | - git config --global user.email "no-reply@noemail.local" 21 | - echo $GH_TOKEN > token.txt 22 | - export GH_TOKEN= 23 | # install gh cli in order to upload artifacts 24 | - curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | gpg --dearmor -o /usr/share/keyrings/githubcli-archive-keyring.gpg 25 | - echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null 26 | - apt update 27 | - apt install gh 28 | build: 29 | commands: 30 | - gh version 31 | - gh auth login --with-token < token.txt 32 | - gh auth status 33 | - | 34 | mvn org.apache.maven.plugins:maven-dependency-plugin:3.1.2:get \ 35 | -DrepoUrl=https://aws.oss.sonatype.org \ 36 | -Dartifact=com.amazonaws:c3r-sdk-core:${VERSION}:jar 37 | - | 38 | mvn org.apache.maven.plugins:maven-dependency-plugin:3.1.2:get \ 39 | -DrepoUrl=https://aws.oss.sonatype.org \ 40 | -Dartifact=com.amazonaws:c3r-sdk-core:${VERSION}:jar:sources 41 | - | 42 | mvn org.apache.maven.plugins:maven-dependency-plugin:3.1.2:get \ 43 | -DrepoUrl=https://aws.oss.sonatype.org \ 44 | -Dartifact=com.amazonaws:c3r-sdk-core:${VERSION}:jar:javadoc 45 | - gh release create v${VERSION} ~/.m2/repository/com/amazonaws/c3r-sdk-core/${VERSION}/*.jar -d -F CHANGELOG.md -t "Cryptographic Computing for Clean Rooms SDK ${VERSION} Release -- $(date +%Y-%m-%d)" -------------------------------------------------------------------------------- /config/checkstyle/license-header.txt: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: Apache-2.0 -------------------------------------------------------------------------------- /config/checkstyle/suppressions.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /config/spotbugs/excludeFilter.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws/c3r/194f528680476c27614fbeb4dbfb0489d62637a2/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.0.1-bin.zip 4 | networkTimeout=10000 5 | zipStoreBase=GRADLE_USER_HOME 6 | zipStorePath=wrapper/dists 7 | -------------------------------------------------------------------------------- /lombok.config: -------------------------------------------------------------------------------- 1 | config.stopBubbling = true 2 | lombok.addLombokGeneratedAnnotation = true 3 | lombok.extern.findbugs.addSuppressFBWarnings = true 4 | lombok.anyConstructor.addConstructorProperties = true -------------------------------------------------------------------------------- /samples/csv/customNull5by6.csv: -------------------------------------------------------------------------------- 1 | cleartext,sealed_none,sealed_max,sealed_fixed,fingerprint_1,fingerprint_2 2 | null,null,null,null,null,null 3 | null,null,null,null,null,null 4 | null,null,null,null,null,null 5 | null,null,null,null,null,null 6 | null,null,null,null,null,null -------------------------------------------------------------------------------- /samples/csv/data_sample_no_headers.csv: -------------------------------------------------------------------------------- 1 | John,Smith,"12345 Mills Rd",Charleston,SC,703-555-1234,CEO,10, 2 | Carl,Carlson,"0 Street",Bethesda,MD,404-555-111,CIO,9,"This is a really long noté that could really be a paragraph" 3 | Jane,Kellog,"1 Two St","New York",NY,304-555-1324,COO,9, 4 | Mark,James,"4 N St",Richmond,VA,407-555-8888,EA,7,"Secret notes" 5 | Jake,Richards,"9 Hollows Rd",Richmond,VA,407-555-1222,"SDE I",4,null 6 | Sean,Hendrix,"8 Hollows Rd",Richmond,VA,407-555-4321,"SDE I",4,"Shana's younger brother" 7 | Shana,Hendrix,"8 Hollows Rd",Richmond,VA,407-555-4322,"SDE II",5,"Sean's older sister" -------------------------------------------------------------------------------- /samples/csv/data_sample_with_quotes.csv: -------------------------------------------------------------------------------- 1 | FirstName,LastName,Address,City,State,PhoneNumber,Title,Level,Notes 2 | John,Smith,"12345 Mills Rd",Charleston,SC,703-555-1234,CEO,10, 3 | Carl,Carlson,"0 Street",Bethesda,MD,404-555-111,CIO,9,"This is a really long noté that could really be a paragraph" 4 | Jane,Kellog,"1 Two St","New York",NY,304-555-1324,COO,9, 5 | Mark,James,"4 N St",Richmond,VA,407-555-8888,EA,7,"Secret notes" 6 | Jake,Richards,"9 Hollows Rd",Richmond,VA,407-555-1222,"SDE I",4,null 7 | Sean,Hendrix,"8 Hollows Rd",Richmond,VA,407-555-4321,"SDE I",4,"Shana's younger brother" 8 | Shana,Hendrix,"8 Hollows Rd",Richmond,VA,407-555-4322,"SDE II",5,"Sean's older sister" -------------------------------------------------------------------------------- /samples/csv/data_sample_without_quotes.csv: -------------------------------------------------------------------------------- 1 | FirstName,LastName,Address,City,State,PhoneNumber,Title,Level,Notes 2 | John,Smith,12345 Mills Rd,Charleston,SC,703-555-1234,CEO,10, 3 | Carl,Carlson,0 Street,Bethesda,MD,404-555-111,CIO,9,This is a really long noté that could really be a paragraph 4 | Jane,Kellog,1 Two St,New York,NY,304-555-1324,COO,9, 5 | Mark,James,4 N St,Richmond,VA,407-555-8888,EA,7,Secret notes 6 | Jake,Richards,9 Hollows Rd,Richmond,VA,407-555-1222,SDE I,4,null 7 | Sean,Hendrix,8 Hollows Rd,Richmond,VA,407-555-4321,SDE I,4,Shana's younger brother 8 | Shana,Hendrix,8 Hollows Rd,Richmond,VA,407-555-4322,SDE II,5,Sean's older sister 9 | -------------------------------------------------------------------------------- /samples/csv/empty5by6.csv: -------------------------------------------------------------------------------- 1 | cleartext,sealed_none,sealed_max,sealed_fixed,fingerprint_1,fingerprint_2 2 | "","","","","","" 3 | "","","","","","" 4 | "","","","","","" 5 | "","","","","","" 6 | "","","","","","" -------------------------------------------------------------------------------- /samples/csv/nonUtf8Encoding.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws/c3r/194f528680476c27614fbeb4dbfb0489d62637a2/samples/csv/nonUtf8Encoding.csv -------------------------------------------------------------------------------- /samples/csv/null5by6.csv: -------------------------------------------------------------------------------- 1 | cleartext,sealed_none,sealed_max,sealed_fixed,fingerprint_1,fingerprint_2 2 | ,,,,, 3 | ,,,,, 4 | ,,,,, 5 | ,,,,, 6 | ,,,,, -------------------------------------------------------------------------------- /samples/csv/one_row_null_sample.csv: -------------------------------------------------------------------------------- 1 | FirstName,LastName,Address,City 2 | John,Smith,INPUT_NULL_VALUE, 3 | -------------------------------------------------------------------------------- /samples/csv/one_row_null_sample_special_null.csv: -------------------------------------------------------------------------------- 1 | FirstName,LastName,Address,City,State,PhoneNumber,Title,Level,Notes 2 | John,Smith,NULL_VALUE,Charleston,SC,703-555-1234,CEO,10, 3 | -------------------------------------------------------------------------------- /samples/csv/one_row_too_few_columns.csv: -------------------------------------------------------------------------------- 1 | FirstName,LastName,Address,City,State,PhoneNumber,Title,Level,Notes 2 | John,Smith,123 Fake St,Charleston,SC, -------------------------------------------------------------------------------- /samples/csv/one_row_too_many_columns.csv: -------------------------------------------------------------------------------- 1 | FirstName,LastName,Address,City,State,PhoneNumber,Title,Level,Notes 2 | John,Smith,123 Fake St,Charleston,SC,703-555-1234,CEO,10,,,,,,, -------------------------------------------------------------------------------- /samples/parquet/all_valid_types.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws/c3r/194f528680476c27614fbeb4dbfb0489d62637a2/samples/parquet/all_valid_types.parquet -------------------------------------------------------------------------------- /samples/parquet/all_valid_types_and_list.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws/c3r/194f528680476c27614fbeb4dbfb0489d62637a2/samples/parquet/all_valid_types_and_list.parquet -------------------------------------------------------------------------------- /samples/parquet/binary_values.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws/c3r/194f528680476c27614fbeb4dbfb0489d62637a2/samples/parquet/binary_values.parquet -------------------------------------------------------------------------------- /samples/parquet/data_sample.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws/c3r/194f528680476c27614fbeb4dbfb0489d62637a2/samples/parquet/data_sample.parquet -------------------------------------------------------------------------------- /samples/parquet/data_sample_with_non_string_types.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws/c3r/194f528680476c27614fbeb4dbfb0489d62637a2/samples/parquet/data_sample_with_non_string_types.parquet -------------------------------------------------------------------------------- /samples/parquet/nonUtf8Encoding.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws/c3r/194f528680476c27614fbeb4dbfb0489d62637a2/samples/parquet/nonUtf8Encoding.parquet -------------------------------------------------------------------------------- /samples/parquet/null_rows_100_groups_1_prim_data.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws/c3r/194f528680476c27614fbeb4dbfb0489d62637a2/samples/parquet/null_rows_100_groups_1_prim_data.parquet -------------------------------------------------------------------------------- /samples/parquet/null_rows_1_groups_1_prim_data.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws/c3r/194f528680476c27614fbeb4dbfb0489d62637a2/samples/parquet/null_rows_1_groups_1_prim_data.parquet -------------------------------------------------------------------------------- /samples/parquet/rows_100_groups_10_prim_data.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws/c3r/194f528680476c27614fbeb4dbfb0489d62637a2/samples/parquet/rows_100_groups_10_prim_data.parquet -------------------------------------------------------------------------------- /samples/parquet/rows_100_groups_1_prim_data.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws/c3r/194f528680476c27614fbeb4dbfb0489d62637a2/samples/parquet/rows_100_groups_1_prim_data.parquet -------------------------------------------------------------------------------- /samples/parquet/rows_1_groups_1_prim_data.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws/c3r/194f528680476c27614fbeb4dbfb0489d62637a2/samples/parquet/rows_1_groups_1_prim_data.parquet -------------------------------------------------------------------------------- /samples/parquet/supported_and_unsupported_types.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws/c3r/194f528680476c27614fbeb4dbfb0489d62637a2/samples/parquet/supported_and_unsupported_types.parquet -------------------------------------------------------------------------------- /samples/schema/6column.json: -------------------------------------------------------------------------------- 1 | { 2 | "headerRow": true, 3 | "columns": [ 4 | { 5 | "sourceHeader": "cleartext", 6 | "type": "cleartext" 7 | }, 8 | { 9 | "sourceHeader": "sealed_none", 10 | "type": "sealed", 11 | "pad": { 12 | "type": "none" 13 | } 14 | }, 15 | { 16 | "sourceHeader": "sealed_max", 17 | "type": "sealed", 18 | "pad": { 19 | "type": "max", 20 | "length": 42 21 | } 22 | }, 23 | { 24 | "sourceHeader": "sealed_fixed", 25 | "type": "sealed", 26 | "pad": { 27 | "type": "fixed", 28 | "length": 42 29 | } 30 | }, 31 | { 32 | "sourceHeader": "fingerprint_1", 33 | "type": "fingerprint" 34 | }, 35 | { 36 | "sourceHeader": "fingerprint_2", 37 | "type": "fingerprint" 38 | } 39 | ] 40 | } 41 | -------------------------------------------------------------------------------- /samples/schema/all_valid_types_and_list_schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "headerRow": true, 3 | "columns": [ 4 | { 5 | "sourceHeader": "strings", 6 | "targetHeader": "strings_cleartext", 7 | "type": "cleartext" 8 | }, 9 | { 10 | "sourceHeader": "strings", 11 | "targetHeader": "strings_fingerprint", 12 | "type": "fingerprint" 13 | }, 14 | { 15 | "sourceHeader": "strings", 16 | "targetHeader": "strings_sealed", 17 | "type": "sealed", 18 | "pad": { 19 | "type": "max", 20 | "length": 0 21 | } 22 | }, 23 | { 24 | "sourceHeader": "small_ints", 25 | "targetHeader": "small_ints_cleartext", 26 | "type": "cleartext" 27 | }, 28 | { 29 | "sourceHeader": "small_ints", 30 | "targetHeader": "small_ints_fingerprint", 31 | "type": "fingerprint" 32 | }, 33 | { 34 | "sourceHeader": "ints", 35 | "targetHeader": "ints_cleartext", 36 | "type": "cleartext" 37 | }, 38 | { 39 | "sourceHeader": "ints", 40 | "targetHeader": "ints_fingerprint", 41 | "type": "fingerprint" 42 | }, 43 | { 44 | "sourceHeader": "big_ints", 45 | "targetHeader": "big_ints_cleartext", 46 | "type": "cleartext" 47 | }, 48 | { 49 | "sourceHeader": "big_ints", 50 | "targetHeader": "big_ints_fingerprint", 51 | "type": "fingerprint" 52 | }, 53 | { 54 | "sourceHeader": "bools", 55 | "targetHeader": "bools_cleartext", 56 | "type": "cleartext" 57 | }, 58 | { 59 | "sourceHeader": "bools", 60 | "targetHeader": "bools_fingerprint", 61 | "type": "fingerprint" 62 | }, 63 | { 64 | "sourceHeader": "dates32", 65 | "targetHeader": "dates32_cleartext", 66 | "type": "cleartext" 67 | }, 68 | { 69 | "sourceHeader": "dates32", 70 | "targetHeader": "dates32_fingerprint", 71 | "type": "fingerprint" 72 | }, 73 | { 74 | "sourceHeader": "dates64", 75 | "targetHeader": "dates64_cleartext", 76 | "type": "cleartext" 77 | }, 78 | { 79 | "sourceHeader": "dates64", 80 | "targetHeader": "dates64_fingerprint", 81 | "type": "fingerprint" 82 | }, 83 | { 84 | "sourceHeader": "binary", 85 | "targetHeader": "binary_cleartext", 86 | "type": "cleartext" 87 | } 88 | ] 89 | } -------------------------------------------------------------------------------- /samples/schema/config_sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "headerRow": true, 3 | "columns": [ 4 | { 5 | "sourceHeader": "FirstName", 6 | "targetHeader": "FirstName", 7 | "type": "cleartext" 8 | }, 9 | { 10 | "sourceHeader": "LastName", 11 | "targetHeader": "LastName", 12 | "type": "cleartext" 13 | }, 14 | { 15 | "sourceHeader": "Address", 16 | "targetHeader": "Address", 17 | "type": "sealed", 18 | "pad": { 19 | "type": "max", 20 | "length": 32 21 | } 22 | }, 23 | { 24 | "sourceHeader": "City", 25 | "targetHeader": "City", 26 | "type": "sealed", 27 | "pad": { 28 | "type": "max", 29 | "length": 16 30 | } 31 | }, 32 | { 33 | "sourceHeader": "State", 34 | "targetHeader": "State", 35 | "type": "fingerprint" 36 | }, 37 | { 38 | "sourceHeader": "PhoneNumber", 39 | "targetHeader": "PhoneNumber_Cleartext", 40 | "type": "cleartext" 41 | }, 42 | { 43 | "sourceHeader": "PhoneNumber", 44 | "targetHeader": "PhoneNumber_Sealed", 45 | "type": "sealed", 46 | "pad": { 47 | "type": "none" 48 | } 49 | }, 50 | { 51 | "sourceHeader": "PhoneNumber", 52 | "targetHeader": "PhoneNumber_Fingerprint", 53 | "type": "fingerprint" 54 | }, 55 | { 56 | "sourceHeader": "Title", 57 | "targetHeader": "Title", 58 | "type": "sealed", 59 | "pad": { 60 | "type": "fixed", 61 | "length": 128 62 | } 63 | }, 64 | { 65 | "sourceHeader": "Level", 66 | "targetHeader": "Level", 67 | "type": "cleartext" 68 | }, 69 | { 70 | "sourceHeader": "Notes", 71 | "targetHeader": "Notes", 72 | "type": "sealed", 73 | "pad": { 74 | "type": "none" 75 | } 76 | } 77 | ] 78 | } 79 | -------------------------------------------------------------------------------- /samples/schema/config_sample_no_cleartext.json: -------------------------------------------------------------------------------- 1 | { 2 | "headerRow": true, 3 | "columns": [ 4 | { 5 | "sourceHeader": "FirstName", 6 | "targetHeader": "FirstName", 7 | "type": "sealed", 8 | "pad": { 9 | "type": "none" 10 | } 11 | }, 12 | { 13 | "sourceHeader": "LastName", 14 | "targetHeader": "LastName", 15 | "type": "sealed", 16 | "pad": { 17 | "type": "none" 18 | } 19 | }, 20 | { 21 | "sourceHeader": "Address", 22 | "targetHeader": "Address", 23 | "type": "sealed", 24 | "pad": { 25 | "type": "max", 26 | "length": 32 27 | } 28 | }, 29 | { 30 | "sourceHeader": "City", 31 | "targetHeader": "City", 32 | "type": "sealed", 33 | "pad": { 34 | "type": "max", 35 | "length": 16 36 | } 37 | }, 38 | { 39 | "sourceHeader": "State", 40 | "targetHeader": "State", 41 | "type": "sealed", 42 | "pad": { 43 | "type": "max", 44 | "length": 32 45 | } 46 | }, 47 | { 48 | "sourceHeader": "PhoneNumber", 49 | "targetHeader": "PhoneNumber_Sealed", 50 | "type": "sealed", 51 | "pad": { 52 | "type": "none" 53 | } 54 | }, 55 | { 56 | "sourceHeader": "PhoneNumber", 57 | "targetHeader": "PhoneNumber_Fingerprint", 58 | "type": "fingerprint" 59 | }, 60 | { 61 | "sourceHeader": "Title", 62 | "targetHeader": "Title", 63 | "type": "sealed", 64 | "pad": { 65 | "type": "fixed", 66 | "length": 128 67 | } 68 | }, 69 | { 70 | "sourceHeader": "Level", 71 | "targetHeader": "Level", 72 | "type": "sealed", 73 | "pad": { 74 | "type": "max", 75 | "length": 32 76 | } 77 | }, 78 | { 79 | "sourceHeader": "Notes", 80 | "targetHeader": "Notes", 81 | "type": "sealed", 82 | "pad": { 83 | "type": "none" 84 | } 85 | } 86 | ] 87 | } 88 | -------------------------------------------------------------------------------- /samples/schema/config_sample_no_headers.json: -------------------------------------------------------------------------------- 1 | { 2 | "headerRow": false, 3 | "columns": [ 4 | [ 5 | { 6 | "targetHeader": "FirstName", 7 | "type": "cleartext" 8 | } 9 | ], 10 | [ 11 | { 12 | "targetHeader": "LastName", 13 | "type": "cleartext" 14 | } 15 | ], 16 | [ 17 | { 18 | "targetHeader": "Address", 19 | "type": "sealed", 20 | "pad": { 21 | "type": "max", 22 | "length": 32 23 | } 24 | } 25 | ], 26 | [ 27 | { 28 | "targetHeader": "City", 29 | "type": "sealed", 30 | "pad": { 31 | "type": "max", 32 | "length": 16 33 | } 34 | } 35 | ], 36 | [ 37 | { 38 | "targetHeader": "State", 39 | "type": "fingerprint" 40 | } 41 | ], 42 | [ 43 | { 44 | "targetHeader": "PhoneNumber_Cleartext", 45 | "type": "cleartext" 46 | }, 47 | { 48 | "targetHeader": "PhoneNumber_Sealed", 49 | "type": "sealed", 50 | "pad": { 51 | "type": "none" 52 | } 53 | }, 54 | { 55 | "targetHeader": "PhoneNumber_Fingerprint", 56 | "type": "fingerprint" 57 | } 58 | ], 59 | [ 60 | { 61 | "targetHeader": "Title", 62 | "type": "sealed", 63 | "pad": { 64 | "type": "fixed", 65 | "length": 128 66 | } 67 | } 68 | ], 69 | [ 70 | { 71 | "targetHeader": "Level", 72 | "type": "cleartext" 73 | } 74 | ], 75 | [ 76 | { 77 | "targetHeader": "Notes", 78 | "type": "sealed", 79 | "pad": { 80 | "type": "none" 81 | } 82 | } 83 | ] 84 | ] 85 | } 86 | -------------------------------------------------------------------------------- /samples/schema/supported_and_unsupported_types.json: -------------------------------------------------------------------------------- 1 | { 2 | "headerRow": true, 3 | "columns": [ 4 | { 5 | "sourceHeader": "strings", 6 | "targetHeader": "strings_cleartext", 7 | "type": "cleartext" 8 | }, 9 | { 10 | "sourceHeader": "strings", 11 | "targetHeader": "strings_fingerprint", 12 | "type": "fingerprint" 13 | }, 14 | { 15 | "sourceHeader": "strings", 16 | "targetHeader": "strings_sealed", 17 | "type": "sealed", 18 | "pad": { 19 | "type": "max", 20 | "length": 0 21 | } 22 | }, 23 | { 24 | "sourceHeader": "small_ints", 25 | "targetHeader": "small_ints_cleartext", 26 | "type": "cleartext" 27 | }, 28 | { 29 | "sourceHeader": "small_ints", 30 | "targetHeader": "small_ints_fingerprint", 31 | "type": "fingerprint" 32 | }, 33 | { 34 | "sourceHeader": "ints", 35 | "targetHeader": "ints_cleartext", 36 | "type": "cleartext" 37 | }, 38 | { 39 | "sourceHeader": "ints", 40 | "targetHeader": "ints_fingerprint", 41 | "type": "fingerprint" 42 | }, 43 | { 44 | "sourceHeader": "big_ints", 45 | "targetHeader": "big_ints_cleartext", 46 | "type": "cleartext" 47 | }, 48 | { 49 | "sourceHeader": "big_ints", 50 | "targetHeader": "big_ints_fingerprint", 51 | "type": "fingerprint" 52 | }, 53 | { 54 | "sourceHeader": "bools", 55 | "targetHeader": "bools_cleartext", 56 | "type": "cleartext" 57 | }, 58 | { 59 | "sourceHeader": "bools", 60 | "targetHeader": "bools_fingerprint", 61 | "type": "fingerprint" 62 | }, 63 | { 64 | "sourceHeader": "dates32", 65 | "targetHeader": "dates32_cleartext", 66 | "type": "cleartext" 67 | }, 68 | { 69 | "sourceHeader": "dates32", 70 | "targetHeader": "dates32_fingerprint", 71 | "type": "fingerprint" 72 | }, 73 | { 74 | "sourceHeader": "dates64", 75 | "targetHeader": "dates64_cleartext", 76 | "type": "cleartext" 77 | }, 78 | { 79 | "sourceHeader": "dates64", 80 | "targetHeader": "dates64_fingerprint", 81 | "type": "fingerprint" 82 | }, 83 | { 84 | "sourceHeader": "binary", 85 | "targetHeader": "binary_cleartext", 86 | "type": "cleartext" 87 | } 88 | ] 89 | } -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This file was generated by the Gradle 'init' task. 3 | * 4 | * The settings file is used to specify which projects to include in your build. 5 | * 6 | * Detailed information about configuring a multi-project build in Gradle can be found 7 | * in the user manual at https://docs.gradle.org/7.5.1/userguide/multi_project_builds.html 8 | * This project uses @Incubating APIs which are subject to change. 9 | */ 10 | 11 | rootProject.name = 'cryptographic-computing-for-clean-rooms' 12 | 13 | include 'c3r-cli' 14 | include 'c3r-cli-spark' 15 | include 'c3r-sdk-core' 16 | include 'c3r-sdk-examples' 17 | include 'c3r-sdk-parquet' -------------------------------------------------------------------------------- /version.txt: -------------------------------------------------------------------------------- 1 | 3.0.5 --------------------------------------------------------------------------------