├── .github ├── 404.html ├── CODEOWNERS ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── documentation-request.md │ ├── feature_request.md │ └── submit-question.md ├── PULL_REQUEST_TEMPLATE.md ├── codecov.yaml └── workflows │ ├── add-to-project.yml │ ├── auto-merge.yml │ ├── blossom-ci.yml │ ├── license-header-check.yml │ ├── markdown-links-check.yml │ ├── markdown-links-check │ └── markdown-links-check-config.json │ ├── mvn-verify-check.yml │ ├── mvn-verify-check │ ├── get-deps-sha1.sh │ └── populate-daily-cache.sh │ ├── pr-description-check.yml │ └── signoff-check.yml ├── .gitignore ├── .gitmodules ├── .pre-commit-config.yaml ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── DF_UDF_README.md ├── LICENSE ├── NOTICE ├── NOTICE-binary ├── README.md ├── SECURITY.md ├── aggregator └── pom.xml ├── api_validation ├── README.md ├── auditAllVersions.sh ├── pom.xml └── src │ └── main │ ├── resources │ └── log4j.properties │ └── scala │ └── com │ └── nvidia │ └── spark │ └── rapids │ └── api │ └── ApiValidation.scala ├── build ├── build-info ├── buildall ├── coverage-report ├── dyn_shim_detection.py ├── get_buildvers.py ├── make-scala-version-build-files.sh └── shimplify.py ├── datagen ├── README.md ├── ScaleTest.md ├── pom.xml └── src │ └── main │ ├── scala │ ├── com │ │ └── nvidia │ │ │ └── rapids │ │ │ └── tests │ │ │ └── scaletest │ │ │ ├── ScaleTestDataGen.scala │ │ │ └── TableGenerator.scala │ └── org │ │ └── apache │ │ └── spark │ │ └── sql │ │ └── tests │ │ └── datagen │ │ └── bigDataGen.scala │ ├── spark320 │ └── scala │ │ └── org │ │ └── apache │ │ └── spark │ │ └── sql │ │ └── tests │ │ └── datagen │ │ └── DataGenExprShims.scala │ └── spark400 │ └── scala │ └── org │ └── apache │ └── spark │ └── sql │ └── tests │ └── datagen │ └── DataGenExprShims.scala ├── delta-lake ├── README.md ├── common │ └── src │ │ └── main │ │ ├── databricks │ │ └── scala │ │ │ ├── com │ │ │ ├── databricks │ │ │ │ └── sql │ │ │ │ │ └── transaction │ │ │ │ │ └── tahoe │ │ │ │ │ └── rapids │ │ │ │ │ ├── GpuCheckDeltaInvariant.scala │ │ │ │ │ ├── GpuDeltaCatalogBase.scala │ │ │ │ │ ├── GpuDeltaInvariantCheckerExec.scala │ │ │ │ │ └── GpuDeltaLog.scala │ │ │ └── nvidia │ │ │ │ └── spark │ │ │ │ └── rapids │ │ │ │ └── delta │ │ │ │ ├── DatabricksDeltaProviderBase.scala │ │ │ │ ├── DeleteCommandMeta.scala │ │ │ │ ├── DeltaWriteUtils.scala │ │ │ │ ├── GpuDeltaParquetFileFormatBase.scala │ │ │ │ ├── MergeIntoCommandMeta.scala │ │ │ │ ├── RapidsDeltaSQLConf.scala │ │ │ │ ├── RapidsDeltaUtils.scala │ │ │ │ ├── UpdateCommandMeta.scala │ │ │ │ └── shims │ │ │ │ └── package-shims.scala │ │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ └── sql │ │ │ └── rapids │ │ │ └── delta │ │ │ ├── GpuIdentityColumn.scala │ │ │ ├── GpuOptimizeWriteExchangeExec.scala │ │ │ └── OptimizeWriteExchangeExec.scala │ │ ├── delta-20x-24x │ │ └── scala │ │ │ ├── com │ │ │ └── nvidia │ │ │ │ └── spark │ │ │ │ └── rapids │ │ │ │ └── delta │ │ │ │ └── Delta20xConfigChecker.scala │ │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ └── sql │ │ │ └── delta │ │ │ ├── rapids │ │ │ ├── GpuOptimisticTransactionBase.scala │ │ │ └── GpuWriteIntoDelta.scala │ │ │ └── shims │ │ │ └── ShimOptimisticTransaction.scala │ │ ├── delta-33x │ │ └── scala │ │ │ ├── com │ │ │ └── nvidia │ │ │ │ └── spark │ │ │ │ └── rapids │ │ │ │ └── delta │ │ │ │ └── DeltaWriteUtils.scala │ │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ └── sql │ │ │ └── delta │ │ │ ├── hooks │ │ │ └── GpuAutoCompact.scala │ │ │ ├── rapids │ │ │ ├── DeltaTrampoline.scala │ │ │ ├── GpuOptimisticTransactionBase.scala │ │ │ ├── GpuOptimizeWriteExchangeExec.scala │ │ │ ├── GpuWriteIntoDelta.scala │ │ │ └── commands │ │ │ │ └── GpuOptimizeExecutor.scala │ │ │ └── shims │ │ │ └── ShimOptimisticTransaction.scala │ │ ├── delta-io │ │ └── scala │ │ │ ├── com │ │ │ └── nvidia │ │ │ │ └── spark │ │ │ │ └── rapids │ │ │ │ └── delta │ │ │ │ ├── DeltaConfigChecker.scala │ │ │ │ ├── DeltaIOProvider.scala │ │ │ │ ├── GpuDeltaDataSource.scala │ │ │ │ ├── GpuDeltaParquetFileFormat.scala │ │ │ │ ├── RapidsDeltaSQLConf.scala │ │ │ │ ├── RapidsDeltaUtils.scala │ │ │ │ └── shims │ │ │ │ ├── ShimDeltaUDF.scala │ │ │ │ └── package-shims.scala │ │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ └── sql │ │ │ ├── delta │ │ │ └── rapids │ │ │ │ ├── AbstractGpuOptimisticTransactionBase.scala │ │ │ │ ├── DeltaRuntimeShim.scala │ │ │ │ ├── GpuCheckDeltaInvariant.scala │ │ │ │ ├── GpuDeltaCatalogBase.scala │ │ │ │ ├── GpuDeltaInvariantCheckerExec.scala │ │ │ │ └── GpuDeltaLog.scala │ │ │ └── rapids │ │ │ └── delta │ │ │ └── GpuIdentityColumn.scala │ │ └── scala │ │ ├── com │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ └── delta │ │ │ ├── DeltaProviderImplBase.scala │ │ │ ├── GpuDeltaIdentityColumnStatsTracker.scala │ │ │ ├── GpuDeltaParquetFileFormatUtils.scala │ │ │ ├── GpuDeltaTaskStatisticsTracker.scala │ │ │ ├── GpuRapidsProcessDeltaMergeJoinExec.scala │ │ │ ├── GpuStatisticsCollection.scala │ │ │ ├── RapidsDeltaWrite.scala │ │ │ └── deltaUDFs.scala │ │ └── org │ │ └── apache │ │ └── spark │ │ └── sql │ │ └── rapids │ │ └── delta │ │ └── DeltaShufflePartitionsUtil.scala ├── delta-20x │ ├── pom.xml │ └── src │ │ └── main │ │ └── scala │ │ ├── com │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ └── delta │ │ │ ├── delta20x │ │ │ ├── DeleteCommandMeta.scala │ │ │ ├── Delta20xProvider.scala │ │ │ ├── GpuDelta20xParquetFileFormat.scala │ │ │ ├── MergeIntoCommandMeta.scala │ │ │ └── UpdateCommandMeta.scala │ │ │ └── shims │ │ │ └── MetadataShims.scala │ │ └── org │ │ └── apache │ │ └── spark │ │ └── sql │ │ └── delta │ │ └── rapids │ │ └── delta20x │ │ ├── Delta20xRuntimeShim.scala │ │ ├── GpuCreateDeltaTableCommand.scala │ │ ├── GpuDeleteCommand.scala │ │ ├── GpuDeltaCatalog.scala │ │ ├── GpuMergeIntoCommand.scala │ │ ├── GpuOptimisticTransaction.scala │ │ └── GpuUpdateCommand.scala ├── delta-21x │ ├── pom.xml │ └── src │ │ └── main │ │ └── scala │ │ ├── com │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ └── delta │ │ │ ├── delta21x │ │ │ ├── DeleteCommandMeta.scala │ │ │ ├── Delta21xProvider.scala │ │ │ ├── GpuDelta21xParquetFileFormat.scala │ │ │ ├── MergeIntoCommandMeta.scala │ │ │ └── UpdateCommandMeta.scala │ │ │ └── shims │ │ │ └── MetadataShims.scala │ │ └── org │ │ └── apache │ │ └── spark │ │ └── sql │ │ └── delta │ │ └── rapids │ │ └── delta21x │ │ ├── Delta21xRuntimeShim.scala │ │ ├── GpuCreateDeltaTableCommand.scala │ │ ├── GpuDeleteCommand.scala │ │ ├── GpuDeltaCatalog.scala │ │ ├── GpuMergeIntoCommand.scala │ │ ├── GpuOptimisticTransaction.scala │ │ └── GpuUpdateCommand.scala ├── delta-22x │ ├── pom.xml │ └── src │ │ └── main │ │ └── scala │ │ ├── com │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ └── delta │ │ │ ├── delta22x │ │ │ ├── DeleteCommandMeta.scala │ │ │ ├── Delta22xProvider.scala │ │ │ ├── GpuDelta22xParquetFileFormat.scala │ │ │ ├── MergeIntoCommandMeta.scala │ │ │ └── UpdateCommandMeta.scala │ │ │ └── shims │ │ │ └── MetadataShims.scala │ │ └── org │ │ └── apache │ │ └── spark │ │ └── sql │ │ └── delta │ │ └── rapids │ │ └── delta22x │ │ ├── Delta22xRuntimeShim.scala │ │ ├── GpuCreateDeltaTableCommand.scala │ │ ├── GpuDeleteCommand.scala │ │ ├── GpuDeltaCatalog.scala │ │ ├── GpuMergeIntoCommand.scala │ │ ├── GpuOptimisticTransaction.scala │ │ └── GpuUpdateCommand.scala ├── delta-23x │ ├── pom.xml │ └── src │ │ └── main │ │ └── scala │ │ ├── com │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ └── delta │ │ │ ├── delta23x │ │ │ ├── DeleteCommandMeta.scala │ │ │ ├── Delta23xProvider.scala │ │ │ ├── GpuDelta23xParquetFileFormat.scala │ │ │ ├── GpuDeltaCatalog.scala │ │ │ ├── MergeIntoCommandMeta.scala │ │ │ └── UpdateCommandMeta.scala │ │ │ └── shims │ │ │ └── MetadataShims.scala │ │ └── org │ │ └── apache │ │ └── spark │ │ └── sql │ │ └── delta │ │ └── rapids │ │ └── delta23x │ │ ├── Delta23xRuntimeShim.scala │ │ ├── GpuCreateDeltaTableCommand.scala │ │ ├── GpuDeleteCommand.scala │ │ ├── GpuMergeIntoCommand.scala │ │ ├── GpuOptimisticTransaction.scala │ │ └── GpuUpdateCommand.scala ├── delta-24x │ ├── pom.xml │ └── src │ │ └── main │ │ └── scala │ │ ├── com │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ └── delta │ │ │ ├── delta24x │ │ │ ├── DeleteCommandMeta.scala │ │ │ ├── Delta24xProvider.scala │ │ │ ├── GpuDelta24xParquetFileFormat.scala │ │ │ ├── GpuDeltaCatalog.scala │ │ │ ├── MergeIntoCommandMeta.scala │ │ │ └── UpdateCommandMeta.scala │ │ │ └── shims │ │ │ └── MetadataShims.scala │ │ └── org │ │ └── apache │ │ └── spark │ │ └── sql │ │ └── delta │ │ └── rapids │ │ └── delta24x │ │ ├── Delta24xRuntimeShim.scala │ │ ├── GpuCreateDeltaTableCommand.scala │ │ ├── GpuDeleteCommand.scala │ │ ├── GpuLowShuffleMergeCommand.scala │ │ ├── GpuMergeIntoCommand.scala │ │ ├── GpuOptimisticTransaction.scala │ │ └── GpuUpdateCommand.scala ├── delta-33x │ ├── pom.xml │ └── src │ │ └── main │ │ └── scala │ │ ├── com │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ └── delta │ │ │ ├── delta33x │ │ │ ├── DeleteCommandMeta.scala │ │ │ ├── Delta33xProvider.scala │ │ │ ├── GpuDelta33xParquetFileFormat.scala │ │ │ ├── GpuDeltaCatalog.scala │ │ │ ├── MergeIntoCommandMeta.scala │ │ │ └── UpdateCommandMeta.scala │ │ │ └── shims │ │ │ └── MetadataShims.scala │ │ └── org │ │ └── apache │ │ └── spark │ │ └── sql │ │ └── delta │ │ └── rapids │ │ └── delta33x │ │ ├── Delta33xRuntimeShim.scala │ │ ├── GpuCreateDeltaTableCommand.scala │ │ ├── GpuDeleteCommand.scala │ │ ├── GpuDeltaFileFormatWriter.scala │ │ ├── GpuMergeIntoCommand.scala │ │ ├── GpuOptimisticTransaction.scala │ │ └── GpuUpdateCommand.scala ├── delta-spark330db │ ├── pom.xml │ └── src │ │ └── main │ │ └── scala │ │ └── com │ │ ├── databricks │ │ └── sql │ │ │ └── transaction │ │ │ └── tahoe │ │ │ └── rapids │ │ │ ├── GpuCreateDeltaTableCommand.scala │ │ │ ├── GpuDeleteCommand.scala │ │ │ ├── GpuDeltaCatalog.scala │ │ │ ├── GpuDeltaDataSource.scala │ │ │ ├── GpuDoAutoCompaction.scala │ │ │ ├── GpuMergeIntoCommand.scala │ │ │ ├── GpuOptimisticTransaction.scala │ │ │ ├── GpuOptimisticTransactionBase.scala │ │ │ ├── GpuOptimizeExecutor.scala │ │ │ ├── GpuUpdateCommand.scala │ │ │ └── GpuWriteIntoDelta.scala │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── delta │ │ ├── DeltaProbe.scala │ │ ├── DeltaSpark330DBProvider.scala │ │ ├── GpuDeltaParquetFileFormat.scala │ │ └── shims │ │ ├── DeleteCommandMetaShim.scala │ │ ├── DeltaLogShim.scala │ │ ├── InvariantViolationExceptionShim.scala │ │ ├── MergeIntoCommandMetaShim.scala │ │ ├── MetadataShims.scala │ │ ├── ShimDeltaUDF.scala │ │ └── UpdateCommandMetaShim.scala ├── delta-spark332db │ ├── pom.xml │ └── src │ │ └── main │ │ └── scala │ │ └── com │ │ ├── databricks │ │ └── sql │ │ │ └── transaction │ │ │ └── tahoe │ │ │ └── rapids │ │ │ ├── GpuCreateDeltaTableCommand.scala │ │ │ ├── GpuDeleteCommand.scala │ │ │ ├── GpuDeltaCatalog.scala │ │ │ ├── GpuDeltaDataSource.scala │ │ │ ├── GpuDoAutoCompaction.scala │ │ │ ├── GpuMergeIntoCommand.scala │ │ │ ├── GpuOptimisticTransaction.scala │ │ │ ├── GpuOptimisticTransactionBase.scala │ │ │ ├── GpuOptimizeExecutor.scala │ │ │ ├── GpuUpdateCommand.scala │ │ │ └── GpuWriteIntoDelta.scala │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── delta │ │ ├── DeltaProbe.scala │ │ ├── DeltaSpark332DBProvider.scala │ │ ├── GpuDeltaParquetFileFormat.scala │ │ └── shims │ │ ├── DeleteCommandMetaShim.scala │ │ ├── DeltaLogShim.scala │ │ ├── InvariantViolationExceptionShim.scala │ │ ├── MergeIntoCommandMetaShim.scala │ │ ├── MetadataShims.scala │ │ ├── ShimDeltaUDF.scala │ │ └── UpdateCommandMetaShim.scala ├── delta-spark341db │ ├── pom.xml │ └── src │ │ └── main │ │ └── scala │ │ └── com │ │ ├── databricks │ │ └── sql │ │ │ └── transaction │ │ │ └── tahoe │ │ │ └── rapids │ │ │ ├── GpuCreateDeltaTableCommand.scala │ │ │ ├── GpuDeleteCommand.scala │ │ │ ├── GpuDeltaCatalog.scala │ │ │ ├── GpuDeltaDataSource.scala │ │ │ ├── GpuDoAutoCompaction.scala │ │ │ ├── GpuLowShuffleMergeCommand.scala │ │ │ ├── GpuMergeIntoCommand.scala │ │ │ ├── GpuOptimisticTransaction.scala │ │ │ ├── GpuOptimisticTransactionBase.scala │ │ │ ├── GpuOptimizeExecutor.scala │ │ │ ├── GpuUpdateCommand.scala │ │ │ └── GpuWriteIntoDelta.scala │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── delta │ │ ├── DeltaProbe.scala │ │ ├── DeltaSpark341DBProvider.scala │ │ ├── GpuDeltaParquetFileFormat.scala │ │ └── shims │ │ ├── DeleteCommandMetaShim.scala │ │ ├── DeltaLogShim.scala │ │ ├── InvariantViolationExceptionShim.scala │ │ ├── MergeIntoCommandMetaShim.scala │ │ ├── MetadataShims.scala │ │ ├── ShimDeltaUDF.scala │ │ └── UpdateCommandMetaShim.scala ├── delta-spark350db143 │ ├── pom.xml │ └── src │ │ └── main │ │ └── scala │ │ └── com │ │ ├── databricks │ │ └── sql │ │ │ └── transaction │ │ │ └── tahoe │ │ │ └── rapids │ │ │ ├── GpuCreateDeltaTableCommand.scala │ │ │ ├── GpuDeleteCommand.scala │ │ │ ├── GpuDeltaCatalog.scala │ │ │ ├── GpuDeltaDataSource.scala │ │ │ ├── GpuDoAutoCompaction.scala │ │ │ ├── GpuMergeIntoCommand.scala │ │ │ ├── GpuOptimisticTransaction.scala │ │ │ ├── GpuOptimisticTransactionBase.scala │ │ │ ├── GpuOptimizeExecutor.scala │ │ │ ├── GpuUpdateCommand.scala │ │ │ └── GpuWriteIntoDelta.scala │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── delta │ │ ├── DeltaProbe.scala │ │ ├── DeltaSpark350DB143Provider.scala │ │ ├── GpuDeltaParquetFileFormat.scala │ │ └── shims │ │ ├── DeleteCommandMetaShim.scala │ │ ├── DeltaLogShim.scala │ │ ├── InvariantViolationExceptionShim.scala │ │ ├── MergeIntoCommandMetaShim.scala │ │ ├── MetadataShims.scala │ │ ├── ShimDeltaUDF.scala │ │ └── UpdateCommandMetaShim.scala └── delta-stub │ ├── pom.xml │ └── src │ └── main │ └── scala │ └── com │ └── nvidia │ └── spark │ └── rapids │ └── delta │ └── DeltaProbeImpl.scala ├── dev └── host_memory_leaks │ ├── README.md │ ├── find_leak.py │ └── leak_detect.cpp ├── dist ├── README.md ├── build │ └── package-parallel-worlds.py ├── maven-antrun │ └── build-parallel-worlds.xml ├── pom.xml ├── scripts │ ├── binary-dedupe.sh │ └── check-shims-revisions.sh ├── unshimmed-common-from-spark320.txt └── unshimmed-from-each-spark3xx.txt ├── docs ├── _config.yml ├── _includes │ └── analytics.html ├── _sass │ └── custom │ │ └── custom.scss ├── additional-functionality │ ├── README.md │ ├── advanced_configs.md │ ├── cache-serializer.md │ ├── rapids-udfs.md │ ├── shuffle-docker-examples │ │ ├── Dockerfile.rocky_no_rdma │ │ ├── Dockerfile.rocky_rdma │ │ ├── Dockerfile.ubuntu_no_rdma │ │ └── Dockerfile.ubuntu_rdma │ └── udf-to-catalyst-expressions.md ├── archive.md ├── archives │ ├── CHANGELOG_0.1_to_0.5.md │ ├── CHANGELOG_21.06_to_21.12.md │ ├── CHANGELOG_22.02_to_22.12.md │ ├── CHANGELOG_23.02_to_23.12.md │ ├── CHANGELOG_24.02-to-24.12.md │ └── CHANGELOG_25.02.md ├── compatibility.md ├── configs.md ├── contact-us.md ├── demo │ ├── AWS-EMR │ │ └── Mortgage-ETL-GPU-EMR.ipynb │ ├── Databricks │ │ └── Mortgage-ETL-db.ipynb │ └── GCP │ │ ├── Mortgage-ETL.ipynb │ │ └── mortgage-xgboost4j-gpu-scala.ipynb ├── dev │ ├── README.md │ ├── adaptive-query.md │ ├── compute_sanitizer.md │ ├── data-sources.md │ ├── get-json-object-dump-tool.md │ ├── gpu-core-dumps.md │ ├── hybrid-execution.md │ ├── idea-code-style-settings.md │ ├── idea-code-style-settings.xml │ ├── lore.md │ ├── mem_debug.md │ ├── microk8s.md │ ├── nvtx_profiling.md │ ├── nvtx_ranges.md │ ├── shimplify.md │ ├── shims.md │ └── testing.md ├── download.md ├── img │ ├── AWS-EMR │ │ ├── EMR_notebook_2.png │ │ ├── EMR_notebook_3.png │ │ ├── bootstrap-action.png │ │ ├── cluster-configuration.png │ │ ├── name-and-applications.png │ │ ├── networking.png │ │ ├── notebook-workspace-creation.png │ │ ├── spark-history-server-UI.png │ │ └── ssh-key-pair.png │ ├── Databricks │ │ ├── a-metric-graph.png │ │ ├── grafana.png │ │ ├── initscript.png │ │ ├── initscript_ucx.png │ │ ├── list-metrics-result.png │ │ ├── list-metrics.png │ │ ├── prometheus-datasource.png │ │ ├── save-prometheus.png │ │ ├── sparkconfig.png │ │ └── sparkconfig_ucx.png │ ├── GCP │ │ ├── dataproc-cluster.png │ │ └── dataproc-service.png │ ├── Tools │ │ ├── qualification-tool-app-view-01.png │ │ ├── qualification-tool-app-view-02.png │ │ ├── qualification-tool-app-view-03.png │ │ └── qualification-tool-recommendation-indexview-with-persql.png │ ├── concurrentPythonWorker.PNG │ ├── join-sql-ui-example.png │ ├── perf-cost.png │ ├── spark3cluster.png │ └── tpcxbb-like-results.png ├── index.md ├── security.md └── supported_ops.md ├── iceberg ├── pom.xml └── src │ └── main │ ├── java │ └── org │ │ └── apache │ │ └── iceberg │ │ ├── data │ │ └── GpuFileHelpers.java │ │ └── spark │ │ └── source │ │ ├── GpuSparkPlanningUtil.java │ │ └── GpuStructInternalRow.java │ └── scala │ ├── com │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── iceberg │ │ ├── IcebergProviderImpl.scala │ │ ├── data │ │ ├── GpuDeleteFilter.scala │ │ └── GpuDeleteLoader.scala │ │ ├── package.scala │ │ ├── parquet │ │ ├── GpuCoalescingIcebergParquetReader.scala │ │ ├── GpuMultiThreadIcebergParquetReader.scala │ │ ├── GpuParquetReaderPostProcessor.scala │ │ ├── GpuSingleThreadIcebergParquetReader.scala │ │ ├── SparkSchemaConverter.scala │ │ ├── converter │ │ │ └── FromIcebergShaded.scala │ │ └── reader.scala │ │ ├── spark │ │ ├── GpuSparkReadOptions.scala │ │ ├── GpuSparkSQLProperties.scala │ │ └── GpuSparkUtil.scala │ │ └── testutils │ │ └── AddEqDeletes.scala │ └── org │ └── apache │ └── iceberg │ ├── parquet │ └── GpuParquetIO.scala │ └── spark │ ├── GpuSparkReadConf.scala │ ├── GpuTypeToSparkType.scala │ └── source │ ├── GpuIcebergPartitionReader.scala │ ├── GpuReaderFactory.scala │ ├── GpuSparkBatch.scala │ ├── GpuSparkBatchQueryScan.scala │ ├── GpuSparkInputPartition.scala │ ├── GpuSparkPartitioningAwareScan.scala │ └── GpuSparkScan.scala ├── integration_tests ├── README.md ├── ScaleTest.md ├── conftest.py ├── pom.xml ├── pytest.ini ├── requirements.txt ├── run_pyspark_from_build.sh ├── runtests.py └── src │ ├── assembly │ └── bin.xml │ ├── main │ ├── java │ │ ├── com │ │ │ └── nvidia │ │ │ │ └── spark │ │ │ │ └── rapids │ │ │ │ └── tests │ │ │ │ └── udf │ │ │ │ └── hive │ │ │ │ ├── EmptyHiveGenericUDF.java │ │ │ │ └── EmptyHiveSimpleUDF.java │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ └── rapids │ │ │ └── tests │ │ │ └── TimeoutSparkListener.java │ ├── python │ │ ├── aqe_test.py │ │ ├── arithmetic_ops_test.py │ │ ├── array_test.py │ │ ├── asserts.py │ │ ├── ast_test.py │ │ ├── avro_test.py │ │ ├── cache_test.py │ │ ├── cast_test.py │ │ ├── cmp_test.py │ │ ├── col_size_exceeding_cudf_limit_test.py │ │ ├── collection_ops_test.py │ │ ├── conditionals_test.py │ │ ├── conftest.py │ │ ├── csv_test.py │ │ ├── data_gen.py │ │ ├── datasourcev2_read_test.py │ │ ├── datasourcev2_write_test.py │ │ ├── date_time_test.py │ │ ├── delta_lake_auto_compact_test.py │ │ ├── delta_lake_delete_test.py │ │ ├── delta_lake_low_shuffle_merge_test.py │ │ ├── delta_lake_merge_common.py │ │ ├── delta_lake_merge_test.py │ │ ├── delta_lake_test.py │ │ ├── delta_lake_time_travel_test.py │ │ ├── delta_lake_update_test.py │ │ ├── delta_lake_utils.py │ │ ├── delta_lake_write_test.py │ │ ├── delta_zorder_test.py │ │ ├── dpp_test.py │ │ ├── expand_exec_test.py │ │ ├── explain_mode_test.py │ │ ├── explain_test.py │ │ ├── fastparquet_compatibility_test.py │ │ ├── fastparquet_utils.py │ │ ├── generate_expr_test.py │ │ ├── get_json_test.py │ │ ├── grouping_sets_test.py │ │ ├── hash_aggregate_test.py │ │ ├── hashing_test.py │ │ ├── higher_order_functions_test.py │ │ ├── hive_delimited_text_test.py │ │ ├── hive_parquet_write_test.py │ │ ├── hive_write_test.py │ │ ├── hybrid_parquet_test.py │ │ ├── hyper_log_log_plus_plus_test.py │ │ ├── iceberg │ │ │ ├── __init__.py │ │ │ ├── iceberg_merge_on_read_test.py │ │ │ └── iceberg_test.py │ │ ├── join_test.py │ │ ├── json_fuzz_test.py │ │ ├── json_matrix_test.py │ │ ├── json_test.py │ │ ├── json_tuple_test.py │ │ ├── kudo_dump_test.py │ │ ├── limit_test.py │ │ ├── logic_test.py │ │ ├── map_test.py │ │ ├── marks.py │ │ ├── misc_expr_test.py │ │ ├── mortgage_test.py │ │ ├── orc_cast_test.py │ │ ├── orc_test.py │ │ ├── orc_write_test.py │ │ ├── parquet_pyarrow_test.py │ │ ├── parquet_test.py │ │ ├── parquet_testing_test.py │ │ ├── parquet_write_test.py │ │ ├── project_lit_alias_test.py │ │ ├── project_presplit_test.py │ │ ├── prune_partition_column_test.py │ │ ├── pyarrow_utils.py │ │ ├── qa_nightly_select_test.py │ │ ├── qa_nightly_sql.py │ │ ├── rand_test.py │ │ ├── range_test.py │ │ ├── regexp_no_unicode_test.py │ │ ├── regexp_test.py │ │ ├── repart_test.py │ │ ├── row-based_udf_test.py │ │ ├── row_conversion_test.py │ │ ├── sample_test.py │ │ ├── scan_default_values_test.py │ │ ├── schema_evolution_test.py │ │ ├── sort_test.py │ │ ├── spark_init_internal.py │ │ ├── spark_session.py │ │ ├── string_test.py │ │ ├── struct_test.py │ │ ├── subquery_test.py │ │ ├── time_window_test.py │ │ ├── timezones.py │ │ ├── udf_cudf_test.py │ │ ├── udf_test.py │ │ ├── url_test.py │ │ └── window_function_test.py │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── tests │ │ ├── DebugRange.scala │ │ ├── datasourcev2 │ │ └── parquet │ │ │ └── TestingV2Source.scala │ │ ├── mortgage │ │ ├── Benchmarks.scala │ │ └── MortgageSpark.scala │ │ └── scaletest │ │ ├── IdleSessionListener.scala │ │ ├── QuerySpecs.scala │ │ ├── ScaleTest.scala │ │ ├── TaskFailureListener.scala │ │ ├── TestReport.scala │ │ └── Utils.scala │ └── test │ ├── resources │ ├── Acquisition_2007Q3.txt │ ├── Performance_2007Q3.txt_0 │ ├── SPARK-32639 │ │ └── 000.snappy.parquet │ ├── V_0_11.orc │ ├── V_0_12.orc │ ├── bad_whitespace.json │ ├── binary_as_string.parquet │ ├── boolean.json │ ├── boolean_formatted.json │ ├── boolean_invalid.json │ ├── date.csv │ ├── dates.json │ ├── dates_invalid.json │ ├── decimal_locale_formatted_strings.json │ ├── decimal_locale_formatted_strings_non_aribic.json │ ├── decimals.csv │ ├── decimals.json │ ├── empty.csv │ ├── empty_arrow.parquet │ ├── empty_int_values.csv │ ├── escaped_strings.json │ ├── fixed-length-char-column-from-hive.orc │ ├── float_formatted.json │ ├── float_formatted_strings.json │ ├── floats.json │ ├── floats_edge_cases.json │ ├── floats_invalid.csv │ ├── floats_invalid.json │ ├── floats_leading_zeros.json │ ├── hive-delim-text │ │ ├── Acquisition_2007Q3 │ │ │ └── Acquisition_2007Q3.hive.txt │ │ ├── Performance_2007Q3 │ │ │ └── Performance_2007Q3.hive.txt │ │ ├── carriage-return-err │ │ │ └── carriage.return.err.hive.txt │ │ ├── carriage-return │ │ │ └── carriage.return.hive.txt │ │ ├── comments │ │ │ └── comments.hive.txt │ │ ├── date │ │ │ └── date.hive.txt │ │ ├── extended-float-values │ │ │ └── ext_float_values.hive.txt │ │ ├── simple-boolean-values │ │ │ └── bools.hive.txt │ │ ├── simple-float-values │ │ │ └── simple_float_values.hive.txt │ │ ├── simple-int-values │ │ │ └── simple_int_values.hive.txt │ │ ├── timestamp │ │ │ └── ts.hive.txt │ │ ├── trucks-1 │ │ │ └── trucks.hive.txt │ │ └── trucks-err │ │ │ └── trucks.err.hive.txt │ ├── int_array_formatted.json │ ├── int_formatted.json │ ├── int_formatted_strings.json │ ├── int_mixed_array_struct_formatted.json │ ├── int_struct_formatted.json │ ├── int_struct_formatted_problematic_rows.json │ ├── ints.json │ ├── ints_invalid.json │ ├── ints_with_whitespace.csv │ ├── invalid_ridealong_columns.json │ ├── just_comments.csv │ ├── log4j.properties │ ├── log4j2.properties │ ├── malformed1.ndjson │ ├── malformed2.ndjson │ ├── malformed3.ndjson │ ├── malformed4.ndjson │ ├── mixed-nested.ndjson │ ├── mixed-primitive-arrays.ndjson │ ├── mixed-primitives-nested.ndjson │ ├── mixed-primitives.ndjson │ ├── mixed-types-in-struct.ndjson │ ├── mixed_objects.json │ ├── nan_and_inf.csv │ ├── nan_and_inf.json │ ├── nan_and_inf_invalid.json │ ├── nan_and_inf_strings.json │ ├── nested-structs.ndjson │ ├── nested_escaped_strings.json │ ├── no-comments.csv │ ├── optional-fields.ndjson │ ├── parquet_acq │ │ └── part-00000-acquisition.snappy.parquet │ ├── parquet_perf │ │ └── part-00000-performance.snappy.parquet │ ├── people.csv │ ├── pytest_log4j.properties │ ├── repeated_columns.json │ ├── scan_emtpy_lines.json │ ├── sci_formatted.json │ ├── sci_formatted_strings.json │ ├── simple-nested.ndjson │ ├── simple_boolean_values.csv │ ├── simple_float_values.csv │ ├── simple_int_values.csv │ ├── single_quoted_strings.json │ ├── small_float_values.csv │ ├── str.csv │ ├── test_orc_varchar.orc │ ├── timestamp-date-test.orc │ ├── timestamp-nanos.parquet │ ├── timestamp_formatted_strings.json │ ├── timestamp_tz_formatted_strings.json │ ├── timestamps.json │ ├── trucks-blank-names.csv │ ├── trucks-comments.csv │ ├── trucks-different.csv │ ├── trucks-empty-values.csv │ ├── trucks-extra-columns.csv │ ├── trucks-missing-quotes.csv │ ├── trucks-more-comments.csv │ ├── trucks-null.csv │ ├── trucks-windows.csv │ ├── trucks.csv │ ├── trucks.tsv │ ├── ts.csv │ ├── unquotedChars.json │ ├── withBackslashEscapingAnyCharacter.json │ ├── withComments.json │ ├── withNonnumericNumbers.json │ ├── withNumericLeadingZeros.json │ ├── withSingleQuotes.json │ ├── withUnquotedControlChars.json │ └── withUnquotedFieldNames.json │ ├── scala │ ├── com │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ ├── functionsSuite.scala │ │ │ └── tests │ │ │ └── mortgage │ │ │ ├── MortgageAdaptiveSparkSuite.scala │ │ │ └── MortgageSparkSuite.scala │ └── org │ │ └── apache │ │ └── spark │ │ └── sql │ │ └── nvidia │ │ └── SparkTestBase.scala │ ├── spark320 │ └── scala │ │ └── org │ │ └── apache │ │ └── spark │ │ └── sql │ │ └── rapids │ │ └── shims │ │ └── TrampolineUtilShim.scala │ └── spark400 │ └── scala │ └── org │ └── apache │ └── spark │ └── sql │ └── rapids │ └── shims │ └── TrampolineUtilShim.scala ├── jdk-profiles └── pom.xml ├── jenkins ├── Dockerfile-blossom.integration.rocky ├── Dockerfile-blossom.integration.ubuntu ├── Dockerfile-blossom.ubuntu ├── Jenkinsfile-blossom.premerge ├── Jenkinsfile-blossom.premerge-databricks ├── databricks │ ├── build.sh │ ├── clusterutils.py │ ├── common_vars.sh │ ├── create.py │ ├── cudf_udf_test.sh │ ├── deploy.sh │ ├── init_cuda11_runtime.sh │ ├── init_cudf_udf.sh │ ├── install_deps.py │ ├── params.py │ ├── run-build.py │ ├── run-tests.py │ ├── run_it.sh │ ├── setup.sh │ ├── shutdown.py │ └── test.sh ├── dependency-check.sh ├── deploy.sh ├── hadoop-def.sh ├── hybrid_execution.sh ├── printJarVersion.sh ├── settings.xml ├── spark-nightly-build.sh ├── spark-premerge-build.sh ├── spark-tests.sh ├── test-timezones.sh └── version-def.sh ├── pom.xml ├── python └── rapids │ ├── __init__.py │ ├── daemon.py │ ├── daemon_databricks.py │ └── worker.py ├── scala2.13 ├── README.md ├── aggregator │ └── pom.xml ├── api_validation │ └── pom.xml ├── datagen │ └── pom.xml ├── delta-lake │ ├── delta-20x │ │ └── pom.xml │ ├── delta-21x │ │ └── pom.xml │ ├── delta-22x │ │ └── pom.xml │ ├── delta-23x │ │ └── pom.xml │ ├── delta-24x │ │ └── pom.xml │ ├── delta-33x │ │ └── pom.xml │ ├── delta-spark330db │ │ └── pom.xml │ ├── delta-spark332db │ │ └── pom.xml │ ├── delta-spark341db │ │ └── pom.xml │ ├── delta-spark350db143 │ │ └── pom.xml │ └── delta-stub │ │ └── pom.xml ├── dist │ └── pom.xml ├── iceberg │ └── pom.xml ├── integration_tests │ └── pom.xml ├── jdk-profiles │ └── pom.xml ├── pom.xml ├── shim-deps │ ├── cloudera │ │ └── pom.xml │ ├── databricks │ │ └── pom.xml │ └── pom.xml ├── shuffle-plugin │ └── pom.xml ├── sql-plugin-api │ └── pom.xml ├── sql-plugin │ └── pom.xml ├── tests │ └── pom.xml ├── tools │ └── pom.xml └── udf-compiler │ └── pom.xml ├── scalastyle-config.xml ├── scripts ├── README.md ├── audit-spark.sh ├── auto-copyrighter.sh ├── generate-changelog └── prioritize-commits.sh ├── shim-deps ├── cloudera │ └── pom.xml ├── databricks │ └── pom.xml └── pom.xml ├── shuffle-plugin ├── pom.xml └── src │ └── main │ └── scala │ └── com │ └── nvidia │ └── spark │ └── rapids │ └── shuffle │ └── ucx │ ├── UCX.scala │ ├── UCXConnection.scala │ ├── UCXShuffleTransport.scala │ └── UCXTransaction.scala ├── sql-plugin-api ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── nvidia │ │ └── spark │ │ ├── RapidsUDF.java │ │ └── Retryable.java │ └── scala │ ├── com │ └── nvidia │ │ └── spark │ │ ├── DFUDFPlugin.scala │ │ ├── ExclusiveModeGpuDiscoveryPlugin.scala │ │ ├── SQLPlugin.scala │ │ ├── functions.scala │ │ ├── rapids │ │ ├── ColumnarRdd.scala │ │ ├── DFUDFPluginAPI.scala │ │ ├── Functions.scala │ │ ├── GpuColumnVectorUtils.scala │ │ ├── GpuCoreDumpMsg.scala │ │ ├── GpuKryoRegistrator.scala │ │ ├── PlanUtils.scala │ │ ├── RapidsShuffleHeartbeatHandler.scala │ │ ├── SQLExecPlugin.scala │ │ ├── ShimLoader.scala │ │ ├── ShimReflectionUtils.scala │ │ ├── SparkShimServiceProvider.scala │ │ └── SparkShimVersion.scala │ │ └── udf │ │ └── Plugin.scala │ └── org │ └── apache │ └── spark │ └── sql │ └── rapids │ ├── ProxyRapidsShuffleInternalManagerBase.scala │ └── execution │ └── UnshimmedTrampolineUtil.scala ├── sql-plugin ├── pom.xml └── src │ ├── main │ ├── format │ │ ├── README.md │ │ ├── ShuffleCommon.fbs │ │ ├── ShuffleMetadataRequest.fbs │ │ ├── ShuffleMetadataResponse.fbs │ │ ├── ShuffleTransferRequest.fbs │ │ └── ShuffleTransferResponse.fbs │ ├── java │ │ ├── com │ │ │ └── nvidia │ │ │ │ └── spark │ │ │ │ └── rapids │ │ │ │ ├── ColumnarCopyHelper.java │ │ │ │ ├── GpuColumnVector.java │ │ │ │ ├── GpuColumnVectorBase.java │ │ │ │ ├── GpuColumnVectorFromBuffer.java │ │ │ │ ├── GpuCompressedColumnVector.java │ │ │ │ ├── GpuPackedTableColumn.java │ │ │ │ ├── HashedPriorityQueue.java │ │ │ │ ├── InternalRowToColumnarBatchIterator.java │ │ │ │ ├── RapidsHostColumnBuilder.java │ │ │ │ ├── RapidsHostColumnVector.java │ │ │ │ ├── RapidsHostColumnVectorCore.java │ │ │ │ ├── RapidsNullSafeHostColumnVector.java │ │ │ │ ├── RapidsNullSafeHostColumnVectorCore.java │ │ │ │ ├── SlicedGpuColumnVector.java │ │ │ │ ├── WithTableBuffer.java │ │ │ │ └── format │ │ │ │ ├── BlockIdMeta.java │ │ │ │ ├── BufferMeta.java │ │ │ │ ├── BufferTransferRequest.java │ │ │ │ ├── BufferTransferResponse.java │ │ │ │ ├── CodecBufferDescriptor.java │ │ │ │ ├── CodecType.java │ │ │ │ ├── MetadataRequest.java │ │ │ │ ├── MetadataResponse.java │ │ │ │ ├── TableMeta.java │ │ │ │ ├── TransferRequest.java │ │ │ │ ├── TransferResponse.java │ │ │ │ └── TransferState.java │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ └── sql │ │ │ └── vectorized │ │ │ └── rapids │ │ │ └── AccessibleArrowColumnVector.java │ ├── scala-2.12 │ │ └── com │ │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ ├── ArmScalaSpecificImpl.scala │ │ │ ├── RapidsStack.scala │ │ │ └── implicits.scala │ ├── scala-2.13 │ │ └── com │ │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ ├── ArmScalaSpecificImpl.scala │ │ │ ├── RapidsStack.scala │ │ │ └── implicits.scala │ ├── scala │ │ ├── com │ │ │ └── nvidia │ │ │ │ └── spark │ │ │ │ ├── CurrentBatchIterator.scala │ │ │ │ ├── DFUDFPluginImpl.scala │ │ │ │ ├── FunctionsImpl.scala │ │ │ │ ├── ParquetCachedBatchSerializer.scala │ │ │ │ ├── TimingUtils.scala │ │ │ │ └── rapids │ │ │ │ ├── AbstractGpuJoinIterator.scala │ │ │ │ ├── AbstractHostByteBufferIterator.scala │ │ │ │ ├── AdaptiveSparkPlanHelperImpl.scala │ │ │ │ ├── Arm.scala │ │ │ │ ├── ArrayIndexUtils.scala │ │ │ │ ├── ArrayUtils.scala │ │ │ │ ├── AstUtil.scala │ │ │ │ ├── AutoClosableArrayBuffer.scala │ │ │ │ ├── AutoCloseColumnBatchIterator.scala │ │ │ │ ├── AvroDataFileReader.scala │ │ │ │ ├── AvroFileWriter.scala │ │ │ │ ├── AvroProvider.scala │ │ │ │ ├── BatchWithPartitionData.scala │ │ │ │ ├── BoolUtils.scala │ │ │ │ ├── BucketJoinTwoSidesPrefetch.scala │ │ │ │ ├── CloseableBufferedIterator.scala │ │ │ │ ├── ColumnCastUtil.scala │ │ │ │ ├── ColumnarFileFormat.scala │ │ │ │ ├── ColumnarOutputWriter.scala │ │ │ │ ├── ColumnarPartitionReaderWithPartitionValues.scala │ │ │ │ ├── CopyCompressionCodec.scala │ │ │ │ ├── CostBasedOptimizer.scala │ │ │ │ ├── DataTypeUtils.scala │ │ │ │ ├── DateUtils.scala │ │ │ │ ├── DecimalUtil.scala │ │ │ │ ├── DeviceMemoryEventHandler.scala │ │ │ │ ├── DumpUtils.scala │ │ │ │ ├── ExplainPlan.scala │ │ │ │ ├── FileUtils.scala │ │ │ │ ├── FloatUtils.scala │ │ │ │ ├── FoldLocalAggregate.scala │ │ │ │ ├── GatherUtils.scala │ │ │ │ ├── GpuAggregateExec.scala │ │ │ │ ├── GpuBatchScanExecMetrics.scala │ │ │ │ ├── GpuBatchUtils.scala │ │ │ │ ├── GpuBoundAttribute.scala │ │ │ │ ├── GpuBringBackToHost.scala │ │ │ │ ├── GpuCSVScan.scala │ │ │ │ ├── GpuCanonicalize.scala │ │ │ │ ├── GpuCast.scala │ │ │ │ ├── GpuCoalesceBatches.scala │ │ │ │ ├── GpuColumnarBatchIterator.scala │ │ │ │ ├── GpuColumnarBatchSerializer.scala │ │ │ │ ├── GpuColumnarToRowExec.scala │ │ │ │ ├── GpuCoreDumpHandler.scala │ │ │ │ ├── GpuDataProducer.scala │ │ │ │ ├── GpuDataTypes.scala │ │ │ │ ├── GpuDataWritingCommandExec.scala │ │ │ │ ├── GpuDeviceManager.scala │ │ │ │ ├── GpuDynamicPruningExpression.scala │ │ │ │ ├── GpuEvalMode.scala │ │ │ │ ├── GpuExec.scala │ │ │ │ ├── GpuExecutedCommandExec.scala │ │ │ │ ├── GpuExpandExec.scala │ │ │ │ ├── GpuExpressions.scala │ │ │ │ ├── GpuGenerateExec.scala │ │ │ │ ├── GpuGetJsonObject.scala │ │ │ │ ├── GpuHashPartitioningBase.scala │ │ │ │ ├── GpuInSet.scala │ │ │ │ ├── GpuJoinUtils.scala │ │ │ │ ├── GpuJsonTuple.scala │ │ │ │ ├── GpuKeyBatchingIterator.scala │ │ │ │ ├── GpuListUtils.scala │ │ │ │ ├── GpuMapUtils.scala │ │ │ │ ├── GpuMetrics.scala │ │ │ │ ├── GpuMonotonicallyIncreasingID.scala │ │ │ │ ├── GpuMultiFileReader.scala │ │ │ │ ├── GpuOrcScan.scala │ │ │ │ ├── GpuOverrides.scala │ │ │ │ ├── GpuParquetFileFormat.scala │ │ │ │ ├── GpuPartitioning.scala │ │ │ │ ├── GpuPostHocResolutionOverrides.scala │ │ │ │ ├── GpuRangePartitioner.scala │ │ │ │ ├── GpuReadCSVFileFormat.scala │ │ │ │ ├── GpuReadFileFormatWithMetrics.scala │ │ │ │ ├── GpuReadOrcFileFormat.scala │ │ │ │ ├── GpuReadParquetFileFormat.scala │ │ │ │ ├── GpuRegExpReplaceMeta.scala │ │ │ │ ├── GpuRoundRobinPartitioning.scala │ │ │ │ ├── GpuRowToColumnarExec.scala │ │ │ │ ├── GpuRunnableCommandExec.scala │ │ │ │ ├── GpuScan.scala │ │ │ │ ├── GpuSemaphore.scala │ │ │ │ ├── GpuShuffleCoalesceExec.scala │ │ │ │ ├── GpuShuffledHashJoinExec.scala │ │ │ │ ├── GpuShuffledSizedHashJoinExec.scala │ │ │ │ ├── GpuSinglePartitioning.scala │ │ │ │ ├── GpuSortExec.scala │ │ │ │ ├── GpuSortMergeJoinMeta.scala │ │ │ │ ├── GpuSparkPartitionID.scala │ │ │ │ ├── GpuTextBasedPartitionReader.scala │ │ │ │ ├── GpuTransitionOverrides.scala │ │ │ │ ├── GpuUserDefinedFunction.scala │ │ │ │ ├── HiveProvider.scala │ │ │ │ ├── HostAlloc.scala │ │ │ │ ├── HostColumnarToGpu.scala │ │ │ │ ├── HostMemoryStreams.scala │ │ │ │ ├── InputFileBlockRule.scala │ │ │ │ ├── InternalExclusiveModeGpuDiscoveryPlugin.scala │ │ │ │ ├── JoinGatherer.scala │ │ │ │ ├── LocationPreservingMapPartitionsRDD.scala │ │ │ │ ├── MapUtil.scala │ │ │ │ ├── MemoryChecker.scala │ │ │ │ ├── MetaUtils.scala │ │ │ │ ├── NvcompLZ4CompressionCodec.scala │ │ │ │ ├── NvcompZSTDCompressionCodec.scala │ │ │ │ ├── NvtxRangeWithDoc.scala │ │ │ │ ├── NvtxWithMetrics.scala │ │ │ │ ├── PartitionReaderIterator.scala │ │ │ │ ├── PlanShims.scala │ │ │ │ ├── Plugin.scala │ │ │ │ ├── PrioritySemaphore.scala │ │ │ │ ├── RangeConfMatcher.scala │ │ │ │ ├── RapidsConf.scala │ │ │ │ ├── RapidsMeta.scala │ │ │ │ ├── RapidsSerializerManager.scala │ │ │ │ ├── RapidsShuffleHeartbeatManager.scala │ │ │ │ ├── RegexComplexityEstimator.scala │ │ │ │ ├── RegexParser.scala │ │ │ │ ├── RmmRapidsRetryIterator.scala │ │ │ │ ├── SamplingUtils.scala │ │ │ │ ├── SaveIntoDataSourceCommandMeta.scala │ │ │ │ ├── ScalableTaskCompletion.scala │ │ │ │ ├── SchemaUtils.scala │ │ │ │ ├── ShimLoaderTemp.scala │ │ │ │ ├── ShuffleBufferCatalog.scala │ │ │ │ ├── ShuffleReceivedBufferCatalog.scala │ │ │ │ ├── SortUtils.scala │ │ │ │ ├── SparkShims.scala │ │ │ │ ├── SpillPriorities.scala │ │ │ │ ├── SpillableColumnarBatch.scala │ │ │ │ ├── SpillableKudoTable.scala │ │ │ │ ├── StrategyRules.scala │ │ │ │ ├── TableCompressionCodec.scala │ │ │ │ ├── TaskRegistryTracker.scala │ │ │ │ ├── ThreadFactoryBuilder.scala │ │ │ │ ├── TypeChecks.scala │ │ │ │ ├── VersionUtils.scala │ │ │ │ ├── basicPhysicalOperators.scala │ │ │ │ ├── conditionalExpressions.scala │ │ │ │ ├── constraintExpressions.scala │ │ │ │ ├── cudf_utils │ │ │ │ └── HostConcatResultUtil.scala │ │ │ │ ├── dataSourceUtil.scala │ │ │ │ ├── datetimeRebaseUtils.scala │ │ │ │ ├── decimalExpressions.scala │ │ │ │ ├── delta │ │ │ │ └── DeltaProvider.scala │ │ │ │ ├── higherOrderFunctions.scala │ │ │ │ ├── hybrid │ │ │ │ ├── HostRetryAllocator.scala │ │ │ │ └── HybridHostRetryAllocator.scala │ │ │ │ ├── iceberg │ │ │ │ └── IcebergProvider.scala │ │ │ │ ├── io │ │ │ │ └── async │ │ │ │ │ ├── AsyncOutputStream.scala │ │ │ │ │ ├── ThrottlingExecutor.scala │ │ │ │ │ └── TrafficController.scala │ │ │ │ ├── jni │ │ │ │ └── kudo │ │ │ │ │ └── KudoHostMergeResultWrapper.scala │ │ │ │ ├── limit.scala │ │ │ │ ├── literals.scala │ │ │ │ ├── lore │ │ │ │ ├── GpuLore.scala │ │ │ │ ├── OutputLoreId.scala │ │ │ │ ├── dump.scala │ │ │ │ ├── package.scala │ │ │ │ └── replay.scala │ │ │ │ ├── namedExpressions.scala │ │ │ │ ├── nullExpressions.scala │ │ │ │ ├── parquet │ │ │ │ ├── GpuParquetScan.scala │ │ │ │ ├── GpuParquetUtils.scala │ │ │ │ ├── ParquetCachedBatchSerializer.scala │ │ │ │ └── ParquetSchemaUtils.scala │ │ │ │ ├── profiler.scala │ │ │ │ ├── python │ │ │ │ ├── GpuPythonArguments.scala │ │ │ │ ├── PythonConfEntries.scala │ │ │ │ └── PythonWorkerSemaphore.scala │ │ │ │ ├── shuffle │ │ │ │ ├── BounceBufferManager.scala │ │ │ │ ├── BufferReceiveState.scala │ │ │ │ ├── BufferSendState.scala │ │ │ │ ├── RapidsShuffleClient.scala │ │ │ │ ├── RapidsShuffleServer.scala │ │ │ │ ├── RapidsShuffleTransport.scala │ │ │ │ └── WindowedBlockIterator.scala │ │ │ │ ├── sparkRapidsListeners.scala │ │ │ │ ├── spill │ │ │ │ └── SpillFramework.scala │ │ │ │ ├── v2WriteCommandMetas.scala │ │ │ │ └── window │ │ │ │ ├── BasicWindowCalc.scala │ │ │ │ ├── GpuBatchedBoundedWindowExec.scala │ │ │ │ ├── GpuBatchedWindowIteratorUtils.scala │ │ │ │ ├── GpuCachedDoublePassWindowExec.scala │ │ │ │ ├── GpuRunningWindowExec.scala │ │ │ │ ├── GpuUnboundedToUnboundedAggWindowExec.scala │ │ │ │ ├── GpuWindowExec.scala │ │ │ │ ├── GpuWindowExecMeta.scala │ │ │ │ └── GpuWindowExpression.scala │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ ├── rapids │ │ │ └── hybrid │ │ │ │ ├── CoalesceConvertIterator.scala │ │ │ │ ├── HybridExecutionUtils.scala │ │ │ │ ├── HybridFileSourceScanExec.scala │ │ │ │ ├── HybridParquetScanRDD.scala │ │ │ │ └── RapidsHostBatchProducer.scala │ │ │ ├── shuffle │ │ │ └── rapids │ │ │ │ └── RapidsShuffleExceptions.scala │ │ │ ├── sql │ │ │ ├── catalyst │ │ │ │ ├── expressions │ │ │ │ │ └── rapids │ │ │ │ │ │ └── Timestamp.scala │ │ │ │ ├── json │ │ │ │ │ └── rapids │ │ │ │ │ │ ├── GpuJsonScan.scala │ │ │ │ │ │ └── GpuReadJsonFileFormat.scala │ │ │ │ └── util │ │ │ │ │ └── rapids │ │ │ │ │ └── DateFormatter.scala │ │ │ ├── execution │ │ │ │ └── datasources │ │ │ │ │ ├── parquet │ │ │ │ │ └── rapids │ │ │ │ │ │ └── ParquetRecordMaterializer.scala │ │ │ │ │ └── rapids │ │ │ │ │ └── OrcFiltersWrapper.scala │ │ │ ├── hive │ │ │ │ └── rapids │ │ │ │ │ ├── GpuHiveFileFormat.scala │ │ │ │ │ ├── GpuHiveOverrides.scala │ │ │ │ │ ├── GpuHiveTableScanExec.scala │ │ │ │ │ ├── GpuHiveTextFileUtils.scala │ │ │ │ │ ├── GpuSaveAsHiveFile.scala │ │ │ │ │ ├── HiveProviderImpl.scala │ │ │ │ │ ├── RapidsHiveErrors.scala │ │ │ │ │ ├── hiveUDFs.scala │ │ │ │ │ └── rowBasedHiveUDFs.scala │ │ │ ├── nvidia │ │ │ │ ├── LogicalPlanRules.scala │ │ │ │ └── dataframe_udfs.scala │ │ │ └── rapids │ │ │ │ ├── AvroProviderImpl.scala │ │ │ │ ├── BasicColumnarWriteStatsTracker.scala │ │ │ │ ├── ColumnarWriteStatsTracker.scala │ │ │ │ ├── ExecutionPlanCaptureCallback.scala │ │ │ │ ├── ExternalSource.scala │ │ │ │ ├── GpuAvroScan.scala │ │ │ │ ├── GpuCartesianProductExec.scala │ │ │ │ ├── GpuDataSourceBase.scala │ │ │ │ ├── GpuDataSourceScanExec.scala │ │ │ │ ├── GpuFileFormatDataWriter.scala │ │ │ │ ├── GpuFileSourceScanExec.scala │ │ │ │ ├── GpuInMemoryTableScanExec.scala │ │ │ │ ├── GpuInputFileBlock.scala │ │ │ │ ├── GpuInsertIntoHadoopFsRelationCommand.scala │ │ │ │ ├── GpuJsonReadCommon.scala │ │ │ │ ├── GpuJsonToStructs.scala │ │ │ │ ├── GpuOrcFileFormat.scala │ │ │ │ ├── GpuParseUrl.scala │ │ │ │ ├── GpuPartitionwiseSampledRDD.scala │ │ │ │ ├── GpuPoissonSampler.scala │ │ │ │ ├── GpuReadAvroFileFormat.scala │ │ │ │ ├── GpuScalaUDF.scala │ │ │ │ ├── GpuScalarSubquery.scala │ │ │ │ ├── GpuShuffleDependency.scala │ │ │ │ ├── GpuShuffleEnv.scala │ │ │ │ ├── GpuStructsToJson.scala │ │ │ │ ├── GpuTaskMetrics.scala │ │ │ │ ├── GpuV1WriteUtils.scala │ │ │ │ ├── GpuWriteStatsTracker.scala │ │ │ │ ├── HashFunctions.scala │ │ │ │ ├── NormalizeFloatingNumbers.scala │ │ │ │ ├── PCBSSchemaHelper.scala │ │ │ │ ├── RapidsDiskBlockManager.scala │ │ │ │ ├── RapidsPrivateUtil.scala │ │ │ │ ├── RapidsShuffleInternalManagerBase.scala │ │ │ │ ├── ShimmedExecutionPlanCaptureCallbackImpl.scala │ │ │ │ ├── TimeWindow.scala │ │ │ │ ├── TimeZoneDB.scala │ │ │ │ ├── aggregate │ │ │ │ ├── GpuApproximatePercentile.scala │ │ │ │ ├── GpuHyperLogLogPlusPlus.scala │ │ │ │ ├── GpuPercentile.scala │ │ │ │ ├── aggregateBase.scala │ │ │ │ └── aggregateFunctions.scala │ │ │ │ ├── arithmetic.scala │ │ │ │ ├── bitwise.scala │ │ │ │ ├── catalyst │ │ │ │ └── expressions │ │ │ │ │ ├── GpuEquivalentExpressions.scala │ │ │ │ │ └── GpuRandomExpressions.scala │ │ │ │ ├── collectionOperations.scala │ │ │ │ ├── complexTypeCreator.scala │ │ │ │ ├── complexTypeExtractors.scala │ │ │ │ ├── datetimeExpressions.scala │ │ │ │ ├── execution │ │ │ │ ├── ExistenceJoin.scala │ │ │ │ ├── GpuBroadcastExchangeExec.scala │ │ │ │ ├── GpuBroadcastHashJoinExecBase.scala │ │ │ │ ├── GpuBroadcastHelper.scala │ │ │ │ ├── GpuBroadcastNestedLoopJoinExecBase.scala │ │ │ │ ├── GpuBroadcastToRowExec.scala │ │ │ │ ├── GpuCustomShuffleReaderExec.scala │ │ │ │ ├── GpuHashJoin.scala │ │ │ │ ├── GpuShuffleExchangeExecBase.scala │ │ │ │ ├── GpuSubPartitionHashJoin.scala │ │ │ │ ├── GpuSubqueryBroadcastExec.scala │ │ │ │ ├── InternalColumnarRddConverter.scala │ │ │ │ ├── SerializedHostTableUtils.scala │ │ │ │ ├── ShuffledBatchRDD.scala │ │ │ │ ├── TrampolineUtil.scala │ │ │ │ └── python │ │ │ │ │ ├── BatchGroupUtils.scala │ │ │ │ │ ├── GpuAggregateInPandasExec.scala │ │ │ │ │ ├── GpuArrowEvalPythonExec.scala │ │ │ │ │ ├── GpuArrowReader.scala │ │ │ │ │ ├── GpuArrowWriter.scala │ │ │ │ │ ├── GpuFlatMapCoGroupsInPandasExec.scala │ │ │ │ │ ├── GpuFlatMapGroupsInPandasExec.scala │ │ │ │ │ ├── GpuMapInBatchExec.scala │ │ │ │ │ ├── GpuMapInPandasExec.scala │ │ │ │ │ ├── GpuPythonExecBase.scala │ │ │ │ │ ├── GpuPythonHelper.scala │ │ │ │ │ ├── GpuPythonRunnerCommon.scala │ │ │ │ │ ├── GpuPythonUDF.scala │ │ │ │ │ └── GpuWindowInPandasExecBase.scala │ │ │ │ ├── mathExpressions.scala │ │ │ │ ├── predicates.scala │ │ │ │ ├── shims │ │ │ │ └── GpuFileScanRDD.scala │ │ │ │ ├── storage │ │ │ │ └── RapidsStorageUtils.scala │ │ │ │ ├── stringFunctions.scala │ │ │ │ ├── test │ │ │ │ └── cpuJsonExpressions.scala │ │ │ │ └── zorder │ │ │ │ ├── GpuHilbertLongIndex.scala │ │ │ │ ├── GpuInterleaveBits.scala │ │ │ │ ├── GpuPartitionerExpr.scala │ │ │ │ └── ZOrderRules.scala │ │ │ └── util │ │ │ └── random │ │ │ └── rapids │ │ │ └── XORShiftRandom.scala │ ├── spark320 │ │ ├── java │ │ │ └── com │ │ │ │ └── nvidia │ │ │ │ └── spark │ │ │ │ └── rapids │ │ │ │ └── shims │ │ │ │ └── ShimSupportsRuntimeFiltering.java │ │ └── scala │ │ │ ├── com │ │ │ └── nvidia │ │ │ │ └── spark │ │ │ │ └── rapids │ │ │ │ ├── RapidsShuffleManager.scala │ │ │ │ ├── shims │ │ │ │ ├── AQEUtils.scala │ │ │ │ ├── AggregationTagging.scala │ │ │ │ ├── AnsiCastRuleShims.scala │ │ │ │ ├── AnsiCastShim.scala │ │ │ │ ├── AnsiUtil.scala │ │ │ │ ├── BatchScanExecMeta.scala │ │ │ │ ├── BloomFilterShims.scala │ │ │ │ ├── BucketSpecForHiveShim.scala │ │ │ │ ├── BucketingUtilsShim.scala │ │ │ │ ├── CastCheckShims.scala │ │ │ │ ├── CastTimeToIntShim.scala │ │ │ │ ├── CastingConfigShim.scala │ │ │ │ ├── CharVarcharUtilsShims.scala │ │ │ │ ├── ColumnDefaultValuesShims.scala │ │ │ │ ├── ConvShim.scala │ │ │ │ ├── CreateDataSourceTableAsSelectCommandMetaShims.scala │ │ │ │ ├── CudfUnsafeRow.scala │ │ │ │ ├── CudfUnsafeRowBase.scala │ │ │ │ ├── DateTimeUtilsShims.scala │ │ │ │ ├── DecimalArithmeticOverrides.scala │ │ │ │ ├── DecimalMultiply128.scala │ │ │ │ ├── DeltaLakeUtils.scala │ │ │ │ ├── DistributionUtil.scala │ │ │ │ ├── FileSourceScanExecMeta.scala │ │ │ │ ├── FilteredPartitions.scala │ │ │ │ ├── GetMapValueMeta.scala │ │ │ │ ├── GetSequenceSize.scala │ │ │ │ ├── GlobalLimitShims.scala │ │ │ │ ├── GpuAggregateInPandasExecMeta.scala │ │ │ │ ├── GpuBatchScanExec.scala │ │ │ │ ├── GpuBatchScanExecBase.scala │ │ │ │ ├── GpuBroadcastJoinMeta.scala │ │ │ │ ├── GpuCastShims.scala │ │ │ │ ├── GpuDataSourceRDD.scala │ │ │ │ ├── GpuFileFormatDataWriterShim.scala │ │ │ │ ├── GpuHashPartitioning.scala │ │ │ │ ├── GpuIntervalUtils.scala │ │ │ │ ├── GpuOptimizedCreateHiveTableAsSelectCommandShims.scala │ │ │ │ ├── GpuOrcDataReader.scala │ │ │ │ ├── GpuOrcDataReader320Plus.scala │ │ │ │ ├── GpuOrcDataReaderBase.scala │ │ │ │ ├── GpuParquetCrypto.scala │ │ │ │ ├── GpuRangePartitioning.scala │ │ │ │ ├── GpuTypeShims.scala │ │ │ │ ├── GpuWindowInPandasExec.scala │ │ │ │ ├── HashUtils.scala │ │ │ │ ├── HybridFileSourceScanExecMeta.scala │ │ │ │ ├── InSubqueryShims.scala │ │ │ │ ├── LegacyBehaviorPolicyShim.scala │ │ │ │ ├── NullIntolerantShim.scala │ │ │ │ ├── NullOutputStreamShim.scala │ │ │ │ ├── OffsetWindowFunctionMeta.scala │ │ │ │ ├── OperatorsUtilShims.scala │ │ │ │ ├── OrcCastingShims.scala │ │ │ │ ├── OrcProtoWriterShim.scala │ │ │ │ ├── OrcReadingShims.scala │ │ │ │ ├── OrcShims.scala │ │ │ │ ├── OrcShims320untilAllBase.scala │ │ │ │ ├── PartitionedFileUtilsShim.scala │ │ │ │ ├── PlanShimsImpl.scala │ │ │ │ ├── PythonUDFShim.scala │ │ │ │ ├── RaiseErrorShim.scala │ │ │ │ ├── RapidsCsvScanMeta.scala │ │ │ │ ├── RapidsFileSourceMetaUtils.scala │ │ │ │ ├── RapidsOrcScanMeta.scala │ │ │ │ ├── RapidsParquetScanMeta.scala │ │ │ │ ├── RebaseShims.scala │ │ │ │ ├── ScanExecShims.scala │ │ │ │ ├── SequenceSizeTooLongErrorBuilder.scala │ │ │ │ ├── ShimAQEShuffleReadExec.scala │ │ │ │ ├── ShimBaseSubqueryExec.scala │ │ │ │ ├── ShimBroadcastExchangeLike.scala │ │ │ │ ├── ShimFilePartitionReaderFactory.scala │ │ │ │ ├── ShimLeafExecNode.scala │ │ │ │ ├── ShimPredicateHelper.scala │ │ │ │ ├── ShuffleManagerShimUtils.scala │ │ │ │ ├── ShuffleOriginUtil.scala │ │ │ │ ├── Spark31Xuntil33XShims.scala │ │ │ │ ├── Spark320PlusNonDBShims.scala │ │ │ │ ├── Spark320PlusShims.scala │ │ │ │ ├── SparkShims.scala │ │ │ │ ├── StaticPartitionShims.scala │ │ │ │ ├── TagScanForRuntimeFiltering.scala │ │ │ │ ├── TreeNode.scala │ │ │ │ ├── TypeSigUtil.scala │ │ │ │ ├── TypeUtilsShims.scala │ │ │ │ ├── XxHash64Shims.scala │ │ │ │ ├── YearParseUtil.scala │ │ │ │ ├── extractValueShims.scala │ │ │ │ ├── gpuWindows.scala │ │ │ │ ├── parquet │ │ │ │ │ ├── ParquetFieldIdShims.scala │ │ │ │ │ ├── ParquetLegacyNanoAsLongShims.scala │ │ │ │ │ ├── ParquetSchemaClipShims.scala │ │ │ │ │ ├── ParquetStringPredShims.scala │ │ │ │ │ └── ParquetTimestampNTZShims.scala │ │ │ │ └── spark320 │ │ │ │ │ └── SparkShimServiceProvider.scala │ │ │ │ ├── shuffle │ │ │ │ └── RapidsShuffleIterator.scala │ │ │ │ └── v1FallbackWriters.scala │ │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ ├── rapids │ │ │ └── shims │ │ │ │ ├── GpuShuffleBlockResolver.scala │ │ │ │ ├── GpuShuffleExchangeExec.scala │ │ │ │ ├── ShuffledBatchRDDUtil.scala │ │ │ │ └── storage │ │ │ │ └── ShimDiskBlockManager.scala │ │ │ ├── sql │ │ │ ├── catalyst │ │ │ │ ├── csv │ │ │ │ │ └── GpuCsvUtils.scala │ │ │ │ └── json │ │ │ │ │ └── GpuJsonUtils.scala │ │ │ ├── errors │ │ │ │ └── ConvUtils.scala │ │ │ ├── execution │ │ │ │ ├── ShimTrampolineUtil.scala │ │ │ │ ├── datasources │ │ │ │ │ ├── parquet │ │ │ │ │ │ ├── ShimCurrentBatchIterator.scala │ │ │ │ │ │ └── rapids │ │ │ │ │ │ │ └── shims │ │ │ │ │ │ │ └── ShimVectorizedColumnReader.scala │ │ │ │ │ ├── rapids │ │ │ │ │ │ └── DataSourceStrategyUtils.scala │ │ │ │ │ └── v2 │ │ │ │ │ │ └── rapids │ │ │ │ │ │ ├── GpuAtomicCreateTableAsSelectExec.scala │ │ │ │ │ │ └── GpuAtomicReplaceTableAsSelectExec.scala │ │ │ │ └── rapids │ │ │ │ │ └── shims │ │ │ │ │ └── FilePartitionShims.scala │ │ │ ├── hive │ │ │ │ └── rapids │ │ │ │ │ └── shims │ │ │ │ │ ├── CommandUtilsShim.scala │ │ │ │ │ ├── FileSinkDescShim.scala │ │ │ │ │ ├── GpuCreateHiveTableAsSelectCommand.scala │ │ │ │ │ ├── GpuInsertIntoHiveTable.scala │ │ │ │ │ ├── GpuRowBasedHiveGenericUDFShim.scala │ │ │ │ │ ├── HiveInspectorsShim.scala │ │ │ │ │ └── HiveProviderCmdShims.scala │ │ │ ├── nvidia │ │ │ │ └── DFUDFShims.scala │ │ │ └── rapids │ │ │ │ ├── GpuDataSource.scala │ │ │ │ ├── GpuFileFormatWriter.scala │ │ │ │ ├── RapidsCachingReader.scala │ │ │ │ ├── RapidsShuffleWriter.scala │ │ │ │ ├── aggregate │ │ │ │ └── aggregateFunctions.scala │ │ │ │ ├── arithmetic.scala │ │ │ │ ├── execution │ │ │ │ ├── GpuBroadcastHashJoinExec.scala │ │ │ │ ├── GpuBroadcastNestedLoopJoinExec.scala │ │ │ │ ├── GpuShuffleMeta.scala │ │ │ │ ├── GpuSubqueryBroadcastMeta.scala │ │ │ │ └── python │ │ │ │ │ └── shims │ │ │ │ │ ├── GpuArrowPythonOutput.scala │ │ │ │ │ ├── GpuArrowPythonRunner.scala │ │ │ │ │ ├── GpuBasePythonRunner.scala │ │ │ │ │ ├── GpuCoGroupedArrowPythonRunner.scala │ │ │ │ │ ├── GpuGroupedPythonRunnerFactory.scala │ │ │ │ │ ├── PythonArgumentUtils.scala │ │ │ │ │ └── WritePythonUDFUtils.scala │ │ │ │ └── shims │ │ │ │ ├── ArrayInvalidArgumentErrorUtils.scala │ │ │ │ ├── ArrowUtilsShim.scala │ │ │ │ ├── AvroUtils.scala │ │ │ │ ├── DataTypeUtilsShim.scala │ │ │ │ ├── GpuAscii.scala │ │ │ │ ├── GpuCastToNumberErrorShim.scala │ │ │ │ ├── GpuCreateDataSourceTableAsSelectCommandShims.scala │ │ │ │ ├── GpuJsonToStructsShim.scala │ │ │ │ ├── GpuMapInPandasExecMeta.scala │ │ │ │ ├── RapidsErrorUtils.scala │ │ │ │ ├── RapidsHadoopWriterUtils.scala │ │ │ │ ├── RapidsQueryErrorUtils.scala │ │ │ │ ├── RapidsShuffleThreadedReader.scala │ │ │ │ ├── RapidsShuffleThreadedWriter.scala │ │ │ │ ├── SchemaUtilsShims.scala │ │ │ │ ├── Spark32XShimsUtils.scala │ │ │ │ ├── SparkSessionUtils.scala │ │ │ │ ├── SparkUpgradeExceptionShims.scala │ │ │ │ ├── TrampolineConnectShims.scala │ │ │ │ ├── datetimeExpressions.scala │ │ │ │ └── misc.scala │ │ │ └── storage │ │ │ ├── RapidsPushBasedFetchHelper.scala │ │ │ └── RapidsShuffleBlockFetcherIterator.scala │ ├── spark321 │ │ └── scala │ │ │ ├── com │ │ │ └── nvidia │ │ │ │ └── spark │ │ │ │ └── rapids │ │ │ │ └── shims │ │ │ │ ├── Spark321PlusShims.scala │ │ │ │ ├── SparkShims.scala │ │ │ │ └── spark321 │ │ │ │ └── SparkShimServiceProvider.scala │ │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ └── sql │ │ │ └── execution │ │ │ └── datasources │ │ │ └── parquet │ │ │ └── rapids │ │ │ └── shims │ │ │ └── ShimVectorizedColumnReader.scala │ ├── spark321cdh │ │ └── scala │ │ │ └── com │ │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ └── shims │ │ │ ├── GpuOrcDataReader.scala │ │ │ ├── GpuParquetCrypto.scala │ │ │ ├── OrcCastingShims.scala │ │ │ ├── OrcShims.scala │ │ │ ├── OrcShims321CDHBase.scala │ │ │ └── spark321cdh │ │ │ └── SparkShimServiceProvider.scala │ ├── spark322 │ │ └── scala │ │ │ └── com │ │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ └── shims │ │ │ ├── SparkShims.scala │ │ │ └── spark322 │ │ │ └── SparkShimServiceProvider.scala │ ├── spark323 │ │ └── scala │ │ │ ├── com │ │ │ └── nvidia │ │ │ │ └── spark │ │ │ │ └── rapids │ │ │ │ └── shims │ │ │ │ └── spark323 │ │ │ │ └── SparkShimServiceProvider.scala │ │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ └── sql │ │ │ └── rapids │ │ │ └── shims │ │ │ └── GpuAscii.scala │ ├── spark324 │ │ └── scala │ │ │ ├── com │ │ │ └── nvidia │ │ │ │ └── spark │ │ │ │ └── rapids │ │ │ │ └── shims │ │ │ │ ├── parquet │ │ │ │ └── ParquetLegacyNanoAsLongShims.scala │ │ │ │ └── spark324 │ │ │ │ └── SparkShimServiceProvider.scala │ │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ └── sql │ │ │ └── rapids │ │ │ └── shims │ │ │ └── RapidsHadoopWriterUtils.scala │ ├── spark330 │ │ └── scala │ │ │ ├── com │ │ │ └── nvidia │ │ │ │ └── spark │ │ │ │ └── rapids │ │ │ │ ├── GpuBloomFilter.scala │ │ │ │ ├── GpuBloomFilterMightContain.scala │ │ │ │ ├── GpuInSubqueryExec.scala │ │ │ │ └── shims │ │ │ │ ├── AnsiUtil.scala │ │ │ │ ├── BloomFilterShims.scala │ │ │ │ ├── BucketSpecForHiveShim.scala │ │ │ │ ├── BucketingUtilsShim.scala │ │ │ │ ├── CastCheckShims.scala │ │ │ │ ├── CharVarcharUtilsShims.scala │ │ │ │ ├── DayTimeIntervalShims.scala │ │ │ │ ├── DistributionUtil.scala │ │ │ │ ├── FilteredPartitions.scala │ │ │ │ ├── GpuBatchScanExec.scala │ │ │ │ ├── GpuDataSourceRDD.scala │ │ │ │ ├── GpuHashPartitioning.scala │ │ │ │ ├── GpuIntervalUtils.scala │ │ │ │ ├── GpuIntervalUtilsBase.scala │ │ │ │ ├── GpuRangePartitioning.scala │ │ │ │ ├── GpuTypeShims.scala │ │ │ │ ├── InSubqueryShims.scala │ │ │ │ ├── OrcReadingShims.scala │ │ │ │ ├── OrcShims.scala │ │ │ │ ├── PythonMapInArrowExecShims.scala │ │ │ │ ├── RapidsFileSourceMetaUtils.scala │ │ │ │ ├── RapidsOrcScanMeta.scala │ │ │ │ ├── RapidsParquetScanMeta.scala │ │ │ │ ├── RoundingShims.scala │ │ │ │ ├── ScanExecShims.scala │ │ │ │ ├── Spark330PlusNonDBShims.scala │ │ │ │ ├── Spark330PlusShims.scala │ │ │ │ ├── SparkShims.scala │ │ │ │ ├── parquet │ │ │ │ ├── ParquetFieldIdShims.scala │ │ │ │ ├── ParquetSchemaClipShims.scala │ │ │ │ └── ParquetTimestampAnnotationShims.scala │ │ │ │ └── spark330 │ │ │ │ └── SparkShimServiceProvider.scala │ │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ └── sql │ │ │ ├── catalyst │ │ │ ├── csv │ │ │ │ └── GpuCsvUtils.scala │ │ │ └── json │ │ │ │ └── GpuJsonUtils.scala │ │ │ ├── execution │ │ │ └── datasources │ │ │ │ ├── parquet │ │ │ │ ├── RapidsVectorizedColumnReader.scala │ │ │ │ ├── ShimCurrentBatchIterator.scala │ │ │ │ └── rapids │ │ │ │ │ └── shims │ │ │ │ │ ├── ParquetCVShims.scala │ │ │ │ │ └── ShimVectorizedColumnReader.scala │ │ │ │ └── v2 │ │ │ │ └── rapids │ │ │ │ ├── GpuAtomicCreateTableAsSelectExec.scala │ │ │ │ └── GpuAtomicReplaceTableAsSelectExec.scala │ │ │ └── rapids │ │ │ ├── aggregate │ │ │ └── GpuBloomFilterAggregate.scala │ │ │ └── shims │ │ │ ├── GpuCastToNumberErrorShim.scala │ │ │ ├── GpuPythonMapInArrowExec.scala │ │ │ ├── GpuPythonMapInArrowExecMeta.scala │ │ │ ├── RapidsErrorUtils.scala │ │ │ ├── RapidsErrorUtils330To334Base.scala │ │ │ ├── RapidsErrorUtilsFor330plus.scala │ │ │ ├── RapidsShuffleThreadedReader.scala │ │ │ ├── SparkUpgradeExceptionShims.scala │ │ │ └── intervalExpressions.scala │ ├── spark330cdh │ │ └── scala │ │ │ ├── com │ │ │ └── nvidia │ │ │ │ └── spark │ │ │ │ └── rapids │ │ │ │ └── shims │ │ │ │ ├── GpuParquetCrypto.scala │ │ │ │ ├── OrcCastingShims.scala │ │ │ │ ├── OrcShims.scala │ │ │ │ ├── Spark33cdhShims.scala │ │ │ │ ├── SparkShims.scala │ │ │ │ └── spark330cdh │ │ │ │ └── SparkShimServiceProvider.scala │ │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ └── sql │ │ │ └── hive │ │ │ └── cdh │ │ │ └── HiveOverrides.scala │ ├── spark330db │ │ └── scala │ │ │ ├── com │ │ │ └── nvidia │ │ │ │ └── spark │ │ │ │ └── rapids │ │ │ │ ├── DatabricksShimServiceProvider.scala │ │ │ │ └── shims │ │ │ │ ├── AQEUtils.scala │ │ │ │ ├── AggregationTagging.scala │ │ │ │ ├── AnsiCastShim.scala │ │ │ │ ├── BatchScanExecMeta.scala │ │ │ │ ├── BucketSpecForHiveShim.scala │ │ │ │ ├── CastingConfigShim.scala │ │ │ │ ├── ColumnDefaultValuesShims.scala │ │ │ │ ├── DecimalArithmeticOverrides.scala │ │ │ │ ├── DeltaLakeUtils.scala │ │ │ │ ├── DistributionUtil.scala │ │ │ │ ├── FileSourceScanExecMeta.scala │ │ │ │ ├── GetMapValueMeta.scala │ │ │ │ ├── GpuBatchScanExec.scala │ │ │ │ ├── GpuBroadcastJoinMeta.scala │ │ │ │ ├── GpuCastShims.scala │ │ │ │ ├── GpuRunningWindowExecMeta.scala │ │ │ │ ├── GpuWindowInPandasExec.scala │ │ │ │ ├── PartitionedFileUtilsShim.scala │ │ │ │ ├── ShimBroadcastExchangeLike.scala │ │ │ │ ├── ShimFilePartitionReaderFactory.scala │ │ │ │ ├── ShimLeafExecNode.scala │ │ │ │ ├── ShuffleOriginUtil.scala │ │ │ │ ├── Spark321PlusDBShims.scala │ │ │ │ ├── Spark330PlusDBShims.scala │ │ │ │ ├── SparkShims.scala │ │ │ │ ├── TypeUtilsShims.scala │ │ │ │ ├── parquet │ │ │ │ ├── ParquetFieldIdShims.scala │ │ │ │ └── ParquetStringPredShims.scala │ │ │ │ ├── shimExpressions.scala │ │ │ │ └── spark330db │ │ │ │ └── SparkShimServiceProvider.scala │ │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ ├── rapids │ │ │ ├── execution │ │ │ │ ├── GpuSubqueryBroadcastMeta.scala │ │ │ │ ├── GpuSubqueryBroadcastMeta330DBBase.scala │ │ │ │ └── shims │ │ │ │ │ └── ReuseGpuBroadcastExchangeAndSubquery.scala │ │ │ └── shims │ │ │ │ ├── GpuDatabricksShuffleExchangeExecBase.scala │ │ │ │ └── GpuShuffleExchangeExec.scala │ │ │ └── sql │ │ │ ├── execution │ │ │ └── datasources │ │ │ │ ├── parquet │ │ │ │ └── rapids │ │ │ │ │ └── shims │ │ │ │ │ ├── ParquetCVShims.scala │ │ │ │ │ └── ShimParquetColumnVector.scala │ │ │ │ ├── rapids │ │ │ │ └── DataSourceStrategyUtils.scala │ │ │ │ └── v2 │ │ │ │ └── rapids │ │ │ │ ├── GpuAtomicCreateTableAsSelectExec.scala │ │ │ │ └── GpuAtomicReplaceTableAsSelectExec.scala │ │ │ └── rapids │ │ │ ├── GpuCheckOverflowInTableInsert.scala │ │ │ ├── aggregate │ │ │ └── aggregateFunctions.scala │ │ │ ├── arithmetic.scala │ │ │ ├── execution │ │ │ ├── GpuBroadcastHashJoinExec.scala │ │ │ ├── GpuBroadcastNestedLoopJoinExec.scala │ │ │ ├── GpuExecutorBroadcastHelper.scala │ │ │ ├── GpuShuffleMeta.scala │ │ │ ├── ShimTrampolineUtil.scala │ │ │ ├── ShuffleExchangeShim.scala │ │ │ └── python │ │ │ │ └── shims │ │ │ │ ├── GpuGroupUDFArrowPythonRunner.scala │ │ │ │ └── GpuGroupedPythonRunnerFactory.scala │ │ │ └── shims │ │ │ ├── RapidsErrorUtils.scala │ │ │ ├── RapidsErrorUtilsBase.scala │ │ │ ├── SparkDateTimeExceptionShims.scala │ │ │ └── SparkUpgradeExceptionShims.scala │ ├── spark331 │ │ └── scala │ │ │ └── com │ │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ └── shims │ │ │ ├── Spark331PlusNonDBShims.scala │ │ │ ├── SparkShims.scala │ │ │ └── spark331 │ │ │ └── SparkShimServiceProvider.scala │ ├── spark332 │ │ └── scala │ │ │ └── com │ │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ └── shims │ │ │ └── spark332 │ │ │ └── SparkShimServiceProvider.scala │ ├── spark332cdh │ │ └── scala │ │ │ └── com │ │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ └── shims │ │ │ ├── SparkShims.scala │ │ │ └── spark332cdh │ │ │ └── SparkShimServiceProvider.scala │ ├── spark332db │ │ └── scala │ │ │ ├── com │ │ │ └── nvidia │ │ │ │ └── spark │ │ │ │ └── rapids │ │ │ │ └── shims │ │ │ │ ├── CreateDataSourceTableAsSelectCommandMetaShims.scala │ │ │ │ ├── GpuBatchScanExec.scala │ │ │ │ ├── GpuFileFormatDataWriterShim.scala │ │ │ │ ├── GpuInsertIntoHiveTable.scala │ │ │ │ ├── GpuKnownNullable.scala │ │ │ │ ├── GpuOptimizedCreateHiveTableAsSelectCommandShims.scala │ │ │ │ ├── LogicalPlanShims.scala │ │ │ │ ├── Spark332PlusDBShims.scala │ │ │ │ ├── SparkShims.scala │ │ │ │ └── spark332db │ │ │ │ └── SparkShimServiceProvider.scala │ │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ └── sql │ │ │ ├── execution │ │ │ └── datasources │ │ │ │ └── GpuWriteFiles.scala │ │ │ ├── hive │ │ │ └── rapids │ │ │ │ └── shims │ │ │ │ ├── CreateFunctions.scala │ │ │ │ ├── GpuRowBasedHiveGenericUDFShim.scala │ │ │ │ ├── HiveFileUtil.scala │ │ │ │ └── HiveProviderCmdShims.scala │ │ │ └── rapids │ │ │ ├── GpuFileFormatWriter.scala │ │ │ └── shims │ │ │ ├── GpuCastToNumberErrorShim.scala │ │ │ ├── GpuCreateDataSourceTableAsSelectCommandShims.scala │ │ │ ├── GpuDataSource.scala │ │ │ ├── SchemaMetadataShims.scala │ │ │ ├── SchemaUtilsShims.scala │ │ │ ├── SparkDateTimeExceptionShims.scala │ │ │ └── SparkUpgradeExceptionShims.scala │ ├── spark333 │ │ └── scala │ │ │ └── com │ │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ └── shims │ │ │ └── spark333 │ │ │ └── SparkShimServiceProvider.scala │ ├── spark334 │ │ └── scala │ │ │ ├── com │ │ │ └── nvidia │ │ │ │ └── spark │ │ │ │ └── rapids │ │ │ │ └── shims │ │ │ │ ├── GetSequenceSize.scala │ │ │ │ └── spark334 │ │ │ │ └── SparkShimServiceProvider.scala │ │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ └── sql │ │ │ └── rapids │ │ │ └── shims │ │ │ ├── RapidsErrorUtils.scala │ │ │ └── SequenceSizeTooLongUnsuccessfulErrorBuilder.scala │ ├── spark340 │ │ └── scala │ │ │ ├── com │ │ │ └── nvidia │ │ │ │ └── spark │ │ │ │ └── rapids │ │ │ │ ├── shims │ │ │ │ ├── BatchScanExecMeta.scala │ │ │ │ ├── CastCheckShims.scala │ │ │ │ ├── ConvShim.scala │ │ │ │ ├── GlobalLimitShims.scala │ │ │ │ ├── GpuBatchScanExec.scala │ │ │ │ ├── GpuBroadcastJoinMeta.scala │ │ │ │ ├── KeyGroupedPartitioningShim.scala │ │ │ │ ├── OrcProtoWriterShim.scala │ │ │ │ ├── PartitionedFileUtilsShim.scala │ │ │ │ ├── PartitionedFileUtilsShimBase.scala │ │ │ │ ├── ShuffleOriginUtil.scala │ │ │ │ ├── Spark340PlusNonDBShims.scala │ │ │ │ ├── SparkShims.scala │ │ │ │ ├── TagScanForRuntimeFiltering.scala │ │ │ │ ├── parquet │ │ │ │ │ ├── ParquetLegacyNanoAsLongShims.scala │ │ │ │ │ ├── ParquetTimestampAnnotationShims.scala │ │ │ │ │ └── ParquetTimestampNTZShims.scala │ │ │ │ └── spark340 │ │ │ │ │ └── SparkShimServiceProvider.scala │ │ │ │ └── shuffle │ │ │ │ └── RapidsShuffleIterator.scala │ │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ └── sql │ │ │ ├── catalyst │ │ │ ├── csv │ │ │ │ └── GpuCsvUtils.scala │ │ │ └── json │ │ │ │ └── GpuJsonUtils.scala │ │ │ ├── errors │ │ │ └── ConvUtils.scala │ │ │ ├── execution │ │ │ └── datasources │ │ │ │ └── v2 │ │ │ │ └── rapids │ │ │ │ ├── GpuAtomicCreateTableAsSelectExec.scala │ │ │ │ └── GpuAtomicReplaceTableAsSelectExec.scala │ │ │ └── rapids │ │ │ ├── RapidsCachingReader.scala │ │ │ ├── execution │ │ │ ├── GpuBroadcastHashJoinExec.scala │ │ │ ├── GpuBroadcastNestedLoopJoinExec.scala │ │ │ └── ShimTrampolineUtil.scala │ │ │ └── shims │ │ │ ├── GpuJsonToStructsShim.scala │ │ │ ├── RapidsErrorUtils.scala │ │ │ └── RapidsErrorUtils340PlusBase.scala │ ├── spark341 │ │ └── scala │ │ │ └── com │ │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ └── shims │ │ │ └── spark341 │ │ │ └── SparkShimServiceProvider.scala │ ├── spark341db │ │ └── scala │ │ │ ├── com │ │ │ └── nvidia │ │ │ │ └── spark │ │ │ │ └── rapids │ │ │ │ └── shims │ │ │ │ ├── GpuAggregateInPandasExecMeta.scala │ │ │ │ ├── GpuToPrettyString.scala │ │ │ │ ├── GpuWindowGroupLimitExec.scala │ │ │ │ ├── PartitionedFileUtilsShim.scala │ │ │ │ ├── PlanShimsImpl.scala │ │ │ │ ├── PythonUDFShim.scala │ │ │ │ ├── Spark341PlusDBShims.scala │ │ │ │ ├── SparkShims.scala │ │ │ │ ├── StaticPartitionShims.scala │ │ │ │ └── spark341db │ │ │ │ └── SparkShimServiceProvider.scala │ │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ ├── rapids │ │ │ └── shims │ │ │ │ └── GpuShuffleExchangeExec.scala │ │ │ └── sql │ │ │ ├── execution │ │ │ ├── datasources │ │ │ │ └── v2 │ │ │ │ │ └── rapids │ │ │ │ │ ├── GpuAtomicCreateTableAsSelectExec.scala │ │ │ │ │ └── GpuAtomicReplaceTableAsSelectExec.scala │ │ │ └── rapids │ │ │ │ └── shims │ │ │ │ ├── FilePartitionShims.scala │ │ │ │ └── SplitFiles.scala │ │ │ ├── hive │ │ │ └── rapids │ │ │ │ └── shims │ │ │ │ ├── CreateFunctions.scala │ │ │ │ ├── FileSinkDescShim.scala │ │ │ │ └── HiveInspectorsShim.scala │ │ │ └── rapids │ │ │ ├── execution │ │ │ ├── ShuffleExchangeShim.scala │ │ │ └── python │ │ │ │ └── shims │ │ │ │ ├── GpuArrowPythonOutput.scala │ │ │ │ ├── GpuArrowPythonRunner.scala │ │ │ │ ├── GpuBasePythonRunner.scala │ │ │ │ ├── GpuCoGroupedArrowPythonRunner.scala │ │ │ │ ├── GpuGroupUDFArrowPythonRunner.scala │ │ │ │ └── GpuGroupedPythonRunnerFactory.scala │ │ │ └── shims │ │ │ ├── RapidsErrorUtils.scala │ │ │ └── RapidsErrorUtils341DBPlusBase.scala │ ├── spark342 │ │ └── scala │ │ │ ├── com │ │ │ └── nvidia │ │ │ │ └── spark │ │ │ │ └── rapids │ │ │ │ └── shims │ │ │ │ ├── DecimalMultiply128.scala │ │ │ │ └── spark342 │ │ │ │ └── SparkShimServiceProvider.scala │ │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ └── sql │ │ │ └── rapids │ │ │ └── shims │ │ │ └── RapidsErrorUtils.scala │ ├── spark343 │ │ └── scala │ │ │ └── com │ │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ └── shims │ │ │ └── spark343 │ │ │ └── SparkShimServiceProvider.scala │ ├── spark344 │ │ └── scala │ │ │ └── com │ │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ └── shims │ │ │ └── spark344 │ │ │ └── SparkShimServiceProvider.scala │ ├── spark350 │ │ └── scala │ │ │ ├── com │ │ │ └── nvidia │ │ │ │ └── spark │ │ │ │ └── rapids │ │ │ │ └── shims │ │ │ │ ├── BatchScanExecMeta.scala │ │ │ │ ├── BatchScanExecMetaBase.scala │ │ │ │ ├── GpuIntervalUtils.scala │ │ │ │ ├── KeyGroupedPartitioningShim.scala │ │ │ │ ├── LegacyBehaviorPolicyShim.scala │ │ │ │ ├── NullOutputStreamShim.scala │ │ │ │ ├── PartitionedFileUtilsShim.scala │ │ │ │ ├── PythonMapInArrowExecShims.scala │ │ │ │ ├── SparkShims.scala │ │ │ │ └── spark350 │ │ │ │ └── SparkShimServiceProvider.scala │ │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ └── sql │ │ │ ├── execution │ │ │ ├── datasources │ │ │ │ ├── parquet │ │ │ │ │ └── rapids │ │ │ │ │ │ └── shims │ │ │ │ │ │ ├── ParquetCVShims.scala │ │ │ │ │ │ └── ShimVectorizedColumnReader.scala │ │ │ │ └── v2 │ │ │ │ │ └── rapids │ │ │ │ │ ├── GpuAtomicCreateTableAsSelectExec.scala │ │ │ │ │ └── GpuAtomicReplaceTableAsSelectExec.scala │ │ │ └── rapids │ │ │ │ └── shims │ │ │ │ └── FilePartitionShims.scala │ │ │ └── rapids │ │ │ ├── execution │ │ │ └── GpuShuffleMeta.scala │ │ │ └── shims │ │ │ ├── ArrowUtilsShim.scala │ │ │ ├── DataTypeUtilsShim.scala │ │ │ ├── GpuMapInPandasExecMeta.scala │ │ │ ├── GpuPythonMapInArrowExecMeta.scala │ │ │ └── SchemaMetadataShims.scala │ ├── spark350db143 │ │ └── scala │ │ │ ├── com │ │ │ └── nvidia │ │ │ │ └── spark │ │ │ │ └── rapids │ │ │ │ ├── RapidsShuffleManager.scala │ │ │ │ └── shims │ │ │ │ ├── BatchScanExecMeta.scala │ │ │ │ ├── CastTimeToIntShim.scala │ │ │ │ ├── CudfUnsafeRow.scala │ │ │ │ ├── DeltaLakeUtils.scala │ │ │ │ ├── GpuBatchScanExec.scala │ │ │ │ ├── PartitionedFileUtilsShim.scala │ │ │ │ ├── PartitionedFileUtilsShimBase.scala │ │ │ │ ├── PythonMapInArrowExecShims.scala │ │ │ │ ├── RaiseErrorShim.scala │ │ │ │ ├── ShuffleManagerShimUtils.scala │ │ │ │ └── spark350db143 │ │ │ │ └── SparkShimServiceProvider.scala │ │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ ├── rapids │ │ │ └── shims │ │ │ │ └── GpuShuffleExchangeExec.scala │ │ │ └── sql │ │ │ ├── execution │ │ │ └── datasources │ │ │ │ └── parquet │ │ │ │ ├── RapidsVectorizedColumnReader.scala │ │ │ │ └── rapids │ │ │ │ └── shims │ │ │ │ ├── ShimParquetColumnVector.scala │ │ │ │ └── ShimVectorizedColumnReader.scala │ │ │ └── rapids │ │ │ ├── RapidsShuffleWriter.scala │ │ │ ├── execution │ │ │ ├── GpuSubqueryBroadcastMeta.scala │ │ │ └── python │ │ │ │ └── shims │ │ │ │ ├── PythonArgumentsUtils.scala │ │ │ │ └── WritePythonUDFUtils.scala │ │ │ └── shims │ │ │ ├── ArrayInvalidArgumentErrorUtils.scala │ │ │ ├── GpuPythonMapInArrowExec.scala │ │ │ ├── GpuPythonMapInArrowExecMeta.scala │ │ │ ├── RapidsErrorUtils.scala │ │ │ ├── SequenceSizeExceededLimitErrorBuilder.scala │ │ │ └── misc.scala │ ├── spark351 │ │ └── scala │ │ │ └── com │ │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ └── shims │ │ │ └── spark351 │ │ │ └── SparkShimServiceProvider.scala │ ├── spark352 │ │ └── scala │ │ │ └── com │ │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ └── shims │ │ │ └── spark352 │ │ │ └── SparkShimServiceProvider.scala │ ├── spark353 │ │ └── scala │ │ │ └── com │ │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ └── shims │ │ │ └── spark353 │ │ │ └── SparkShimServiceProvider.scala │ ├── spark354 │ │ └── scala │ │ │ └── com │ │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ └── shims │ │ │ └── spark354 │ │ │ └── SparkShimServiceProvider.scala │ ├── spark355 │ │ └── scala │ │ │ ├── com │ │ │ └── nvidia │ │ │ │ └── spark │ │ │ │ └── rapids │ │ │ │ └── shims │ │ │ │ ├── PartitionedFileUtilsShim.scala │ │ │ │ └── spark355 │ │ │ │ └── SparkShimServiceProvider.scala │ │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ └── sql │ │ │ └── execution │ │ │ └── rapids │ │ │ └── shims │ │ │ └── FilePartitionShims.scala │ ├── spark356 │ │ └── scala │ │ │ ├── com │ │ │ └── nvidia │ │ │ │ └── spark │ │ │ │ └── rapids │ │ │ │ └── shims │ │ │ │ └── spark356 │ │ │ │ └── SparkShimServiceProvider.scala │ │ │ └── org │ │ │ └── apache │ │ │ └── spark │ │ │ └── sql │ │ │ └── execution │ │ │ └── datasources │ │ │ └── v2 │ │ │ └── rapids │ │ │ ├── GpuAtomicCreateTableAsSelectExec.scala │ │ │ └── GpuAtomicReplaceTableAsSelectExec.scala │ └── spark400 │ │ └── scala │ │ ├── com │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ └── shims │ │ │ ├── CudfUnsafeRow.scala │ │ │ ├── DateTimeUtilsShims.scala │ │ │ ├── GpuOrcDataReader.scala │ │ │ ├── LogicalPlanShims.scala │ │ │ ├── MapInArrowExecShims.scala │ │ │ ├── NullIntolerantShim.scala │ │ │ ├── OperatorsUtilShims.scala │ │ │ ├── PartitionedFileUtilsShim.scala │ │ │ └── spark400 │ │ │ └── SparkShimServiceProvider.scala │ │ └── org │ │ └── apache │ │ └── spark │ │ └── sql │ │ ├── execution │ │ ├── datasources │ │ │ └── v2 │ │ │ │ └── rapids │ │ │ │ ├── GpuAtomicCreateTableAsSelectExec.scala │ │ │ │ └── GpuAtomicReplaceTableAsSelectExec.scala │ │ └── rapids │ │ │ └── shims │ │ │ └── FilePartitionShims.scala │ │ ├── hive │ │ └── rapids │ │ │ └── shims │ │ │ └── CommandUtilsShim.scala │ │ ├── nvidia │ │ └── DFUDFShims.scala │ │ └── rapids │ │ ├── execution │ │ ├── GpuSubqueryBroadcastMeta.scala │ │ └── python │ │ │ └── shims │ │ │ ├── GpuBasePythonRunner.scala │ │ │ └── WritePythonUDFUtils.scala │ │ └── shims │ │ ├── GpuMapInArrowExecMeta.scala │ │ ├── RapidsErrorUtils.scala │ │ ├── SparkSessionUtils.scala │ │ └── TrampolineConnectShims.scala │ └── test │ ├── resources │ ├── log4j.properties │ └── log4j2.properties │ ├── scala │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ ├── ArmSuite.scala │ │ ├── FQSuiteName.scala │ │ ├── GpuDeviceManagerSuite.scala │ │ ├── HostAllocSuite.scala │ │ ├── RangeConfMatcherSuite.scala │ │ ├── ThreadFactoryBuilderTest.scala │ │ └── io │ │ └── async │ │ ├── AsyncOutputStreamSuite.scala │ │ ├── ThrottlingExecutorSuite.scala │ │ └── TrafficControllerSuite.scala │ ├── spark320 │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── spark320 │ │ └── SparkShimsSuite.scala │ ├── spark321 │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── spark321 │ │ └── SparkShimsSuite.scala │ ├── spark321cdh │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── spark321cdh │ │ └── SparkShimsSuite.scala │ ├── spark322 │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── spark322 │ │ └── SparkShimsSuite.scala │ ├── spark323 │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── spark323 │ │ └── SparkShimsSuite.scala │ ├── spark324 │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── spark324 │ │ └── SparkShimsSuite.scala │ ├── spark330 │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── spark330 │ │ └── SparkShimsSuite.scala │ ├── spark330cdh │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── spark330cdh │ │ └── SparkShimsSuite.scala │ ├── spark331 │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── spark331 │ │ └── SparkShimsSuite.scala │ ├── spark332 │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── spark332 │ │ └── SparkShimsSuite.scala │ ├── spark332cdh │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── spark332cdh │ │ └── SparkShimsSuite.scala │ ├── spark333 │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── spark333 │ │ └── SparkShimsSuite.scala │ ├── spark334 │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── spark334 │ │ └── SparkShimsSuite.scala │ ├── spark340 │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── spark340 │ │ └── SparkShimsSuite.scala │ ├── spark341 │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── spark341 │ │ └── SparkShimsSuite.scala │ ├── spark342 │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── spark342 │ │ └── SparkShimsSuite.scala │ ├── spark343 │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── spark343 │ │ └── SparkShimsSuite.scala │ ├── spark344 │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── spark344 │ │ └── SparkShimsSuite.scala │ ├── spark350 │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── spark350 │ │ └── SparkShimsSuite.scala │ ├── spark351 │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── spark351 │ │ └── SparkShimsSuite.scala │ ├── spark352 │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── spark352 │ │ └── SparkShimsSuite.scala │ ├── spark353 │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── spark353 │ │ └── SparkShimsSuite.scala │ ├── spark354 │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── spark354 │ │ └── SparkShimsSuite.scala │ ├── spark355 │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── spark355 │ │ └── SparkShimsSuite.scala │ ├── spark356 │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── spark356 │ │ └── SparkShimsSuite.scala │ └── spark400 │ └── scala │ └── com │ └── nvidia │ └── spark │ └── rapids │ └── shims │ └── spark400 │ └── SparkShimsSuite.scala ├── tests ├── README.md ├── pom.xml └── src │ └── test │ ├── java │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ ├── TestHashedPriorityQueue.java │ │ └── TestStats.java │ ├── resources │ ├── dates.csv │ ├── day-time-interval-to-be-fix.csv │ ├── day-time-interval.csv │ ├── decimal-test-legacy.parquet │ ├── decimal-test.csv │ ├── decimal-test.orc │ ├── decimal-test.parquet │ ├── disorder-read-schema.parquet │ ├── file-splits.orc │ ├── file-splits.parquet │ ├── floats.csv │ ├── hybrid-Julian-calendar.orc │ ├── ints.csv │ ├── log4j.properties │ ├── log4j2.properties │ ├── lots_o_longs.csv │ ├── map_of_strings.snappy.parquet │ ├── mixed-types.csv │ ├── nested-unsigned-malformed.parquet │ ├── nested-unsigned.parquet │ ├── nullable_floats.csv │ ├── partitioned-csv │ │ ├── partKey=100 │ │ │ └── part-00000-e2c10641-1db9-47a3-be72-a1a5e94f63c6.c000.csv │ │ └── partKey=102 │ │ │ └── part-00000-e2c10641-1db9-47a3-be72-a1a5e94f63c6.c000.csv │ ├── partitioned-parquet │ │ ├── partKey=100 │ │ │ └── part-00000-7404bd47-cd68-43cf-b78e-5f1c5fba912d.c000.snappy.parquet │ │ └── partKey=102 │ │ │ └── part-00000-7404bd47-cd68-43cf-b78e-5f1c5fba912d.c000.snappy.parquet │ ├── schema-can-prune.orc │ ├── schema-cant-prune.orc │ ├── shorts.csv │ ├── strings.csv │ ├── test.csv │ ├── test_unsigned64.parquet │ ├── test_withheaders.csv │ ├── timestamp-date-test-msec.parquet │ ├── timestamp-date-test.parquet │ ├── timestamps.csv │ ├── very_large_longs.csv │ ├── window-function-test-nonull.orc │ └── window-function-test.orc │ ├── scala │ ├── com │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ ├── AdaptiveQueryExecSuite.scala │ │ │ ├── AnsiCastOpSuite.scala │ │ │ ├── ApproximatePercentileSuite.scala │ │ │ ├── AstUtilSuite.scala │ │ │ ├── BatchWithPartitionDataSuite.scala │ │ │ ├── BroadcastHashJoinSuite.scala │ │ │ ├── BroadcastNestedLoopJoinSuite.scala │ │ │ ├── CachedBatchWriterSuite.scala │ │ │ ├── CastOpSuite.scala │ │ │ ├── CollectionOpSuite.scala │ │ │ ├── ColumnToRowIteratorRetrySuite.scala │ │ │ ├── ConditionalsSuite.scala │ │ │ ├── CostBasedOptimizerSuite.scala │ │ │ ├── CsvScanRetrySuite.scala │ │ │ ├── CsvScanSuite.scala │ │ │ ├── CudfTestHelper.scala │ │ │ ├── DecimalBinaryOpSuite.scala │ │ │ ├── DeviceMemoryEventHandlerSuite.scala │ │ │ ├── DistributionSuite.scala │ │ │ ├── DumpUtilsSuite.scala │ │ │ ├── ExpandExecSuite.scala │ │ │ ├── FilterExprSuite.scala │ │ │ ├── FunSuiteWithTempDir.scala │ │ │ ├── FuzzerUtils.scala │ │ │ ├── GeneratedInternalRowToCudfRowIteratorRetrySuite.scala │ │ │ ├── GpuArchitectureTestSuite.scala │ │ │ ├── GpuBatchUtilsSuite.scala │ │ │ ├── GpuBringBackToHostSuite.scala │ │ │ ├── GpuCoalesceBatchesRetrySuite.scala │ │ │ ├── GpuCoalesceBatchesSuite.scala │ │ │ ├── GpuColumnarToRowSuite.scala │ │ │ ├── GpuCoreDumpHandlerSuite.scala │ │ │ ├── GpuDeviceManagerSuite.scala │ │ │ ├── GpuExpressionTestSuite.scala │ │ │ ├── GpuFileScanPrunePartitionSuite.scala │ │ │ ├── GpuGenerateSuite.scala │ │ │ ├── GpuKryoRegistratorSuite.scala │ │ │ ├── GpuMultiFileReaderSuite.scala │ │ │ ├── GpuPartitioningSuite.scala │ │ │ ├── GpuReaderTypeSuites.scala │ │ │ ├── GpuSemaphoreSuite.scala │ │ │ ├── GpuShuffledHashJoinExecSuite.scala │ │ │ ├── GpuSinglePartitioningSuite.scala │ │ │ ├── GpuSortRetrySuite.scala │ │ │ ├── GpuSubPartitionSuite.scala │ │ │ ├── GpuUnitTests.scala │ │ │ ├── HashAggregateRetrySuite.scala │ │ │ ├── HashAggregatesSuite.scala │ │ │ ├── HashSortOptimizeSuite.scala │ │ │ ├── HiveHashTestSuite.scala │ │ │ ├── HostColumnToGpuRetrySuite.scala │ │ │ ├── ImplicitsTestSuite.scala │ │ │ ├── InsertPartitionSuite.scala │ │ │ ├── JoinsSuite.scala │ │ │ ├── JsonScanRetrySuite.scala │ │ │ ├── KnownNotNullSuite.scala │ │ │ ├── LimitExecSuite.scala │ │ │ ├── LimitRetrySuite.scala │ │ │ ├── LogOperatorUnitTestSuite.scala │ │ │ ├── MetaUtilsSuite.scala │ │ │ ├── MetricsSuite.scala │ │ │ ├── NonDeterministicRetrySuite.scala │ │ │ ├── OrcQuerySuite.scala │ │ │ ├── OrcScanSuite.scala │ │ │ ├── OrcSuite.scala │ │ │ ├── ParquetCompressionCodecSuite.scala │ │ │ ├── ParquetFieldIdSuite.scala │ │ │ ├── ParquetQuerySuite.scala │ │ │ ├── ParquetScaleTestSuite.scala │ │ │ ├── ParquetScaleTestUtils.scala │ │ │ ├── ParquetScanSuite.scala │ │ │ ├── ParquetWriterSuite.scala │ │ │ ├── ParseDateTimeSuite.scala │ │ │ ├── PrioritySemaphoreSuite.scala │ │ │ ├── RangeRetrySuite.scala │ │ │ ├── RapidsExecutorPluginSuite.scala │ │ │ ├── RapidsHostColumnBuilderSuite.scala │ │ │ ├── RebaseHelperSuite.scala │ │ │ ├── RegularExpressionParserSuite.scala │ │ │ ├── RegularExpressionRewriteSuite.scala │ │ │ ├── RegularExpressionSuite.scala │ │ │ ├── RegularExpressionTranspilerSuite.scala │ │ │ ├── ReusedExchangeFixupSuite.scala │ │ │ ├── RmmSparkRetrySuiteBase.scala │ │ │ ├── RowBasedUDFSuite.scala │ │ │ ├── RowToColumnarIteratorRetrySuite.scala │ │ │ ├── ScalaUDFSuite.scala │ │ │ ├── ScalableTaskCompletionSuite.scala │ │ │ ├── ScalarSubquerySuite.scala │ │ │ ├── SerializationSuite.scala │ │ │ ├── ShuffleBufferCatalogSuite.scala │ │ │ ├── ShufflePartitionerRetrySuite.scala │ │ │ ├── SortExecSuite.scala │ │ │ ├── SparkQueryCompareTestSuite.scala │ │ │ ├── StringFunctionSuite.scala │ │ │ ├── TestUtils.scala │ │ │ ├── TimeOperatorsSuite.scala │ │ │ ├── UnaryOperatorsSuite.scala │ │ │ ├── WindowFunctionSuite.scala │ │ │ ├── WindowRetrySuite.scala │ │ │ ├── WithRetrySuite.scala │ │ │ ├── hybrid │ │ │ └── HostRetryAllocatorTest.scala │ │ │ ├── lore │ │ │ ├── GpuLoreSuite.scala │ │ │ └── OutputLoreIdSuite.scala │ │ │ ├── shuffle │ │ │ ├── RapidsShuffleClientSuite.scala │ │ │ ├── RapidsShuffleHeartbeatManagerSuite.scala │ │ │ ├── RapidsShuffleIteratorSuite.scala │ │ │ ├── RapidsShuffleServerSuite.scala │ │ │ ├── UCXConnectionSuite.scala │ │ │ └── WindowedBlockIteratorSuite.scala │ │ │ ├── spill │ │ │ └── SpillFrameworkSuite.scala │ │ │ ├── tests │ │ │ ├── CacheTestSuite.scala │ │ │ └── udf │ │ │ │ └── scala │ │ │ │ ├── AlwaysTrueUDF.scala │ │ │ │ ├── URLDecode.scala │ │ │ │ └── URLEncode.scala │ │ │ ├── timezone │ │ │ ├── TimeZonePerfSuite.scala │ │ │ └── TimeZoneSuite.scala │ │ │ ├── unit │ │ │ ├── ComplexCreatorSizeEstimationTest.scala │ │ │ ├── DateTimeUnitTest.scala │ │ │ ├── DecimalUnitTest.scala │ │ │ ├── GpuScalarUnitTest.scala │ │ │ ├── LiteralSizeEstimationTest.scala │ │ │ └── StringRepeatUnitTest.scala │ │ │ └── window │ │ │ └── GpuUnboundedToUnboundedAggWindowSuite.scala │ └── org │ │ └── apache │ │ └── spark │ │ └── sql │ │ ├── FileUtils.scala │ │ ├── GpuSparkPlanSuite.scala │ │ ├── rapids │ │ ├── BroadcastLocalPropertySuite.scala │ │ ├── CanonicalizeSuite.scala │ │ ├── GpuFileFormatDataWriterSuite.scala │ │ ├── MyDenseVectorUDT.scala │ │ ├── OrcFilterSuite.scala │ │ ├── ParquetFilterSuite.scala │ │ ├── ParquetFormatScanSuite.scala │ │ ├── ProjectExprSuite.scala │ │ ├── SpillableColumnarBatchSuite.scala │ │ ├── TestTrampolineUtil.scala │ │ ├── TestingUDT.scala │ │ ├── catalyst │ │ │ └── expressions │ │ │ │ └── GpuEquivalentExpressionsSuite.scala │ │ ├── execution │ │ │ ├── InternalColumnarRDDConverterSuite.scala │ │ │ └── python │ │ │ │ └── GpuPythonHelperSuite.scala │ │ ├── filecache │ │ │ ├── FileCacheIntegrationSuite.scala │ │ │ └── NoFileCacheIntegrationSuite.scala │ │ └── metrics │ │ │ └── source │ │ │ └── MockTaskContext.scala │ │ └── timezone │ │ └── TimeZonePerfUtils.scala │ ├── spark320 │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ ├── ConcurrentWriterMetricsSuite.scala │ │ ├── shims │ │ └── OrcStatisticShim.scala │ │ └── shuffle │ │ └── RapidsShuffleTestHelper.scala │ ├── spark321 │ └── scala │ │ ├── com │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ ├── DynamicPruningSuite.scala │ │ │ └── OrcEncryptionSuite.scala │ │ └── org │ │ └── apache │ │ └── spark │ │ └── sql │ │ └── rapids │ │ ├── RapidsShuffleThreadedReaderSuite.scala │ │ └── RapidsShuffleThreadedWriterSuite.scala │ ├── spark321cdh │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shims │ │ └── OrcStatisticShim.scala │ ├── spark330 │ └── scala │ │ ├── com │ │ └── nvidia │ │ │ └── spark │ │ │ └── rapids │ │ │ ├── BloomFilterAggregateQuerySuite.scala │ │ │ ├── CsvScanForIntervalSuite.scala │ │ │ ├── GpuIntervalUtilsTest.scala │ │ │ ├── IntervalArithmeticSuite.scala │ │ │ ├── IntervalCastSuite.scala │ │ │ ├── IntervalDivisionSuite.scala │ │ │ ├── IntervalMultiplySuite.scala │ │ │ ├── IntervalSuite.scala │ │ │ ├── ParquetUDTSuite.scala │ │ │ ├── SampleSuite.scala │ │ │ └── TimestampSuite.scala │ │ └── org │ │ └── apache │ │ └── spark │ │ └── sql │ │ └── rapids │ │ ├── GpuInSubqueryExecSuite.scala │ │ ├── suites │ │ ├── RapidsCastSuite.scala │ │ ├── RapidsDataFrameAggregateSuite.scala │ │ ├── RapidsJsonExpressionsSuite.scala │ │ ├── RapidsJsonFunctionsSuite.scala │ │ ├── RapidsJsonSuite.scala │ │ ├── RapidsMathFunctionsSuite.scala │ │ ├── RapidsParquetAvroCompatibilitySuite.scala │ │ ├── RapidsParquetColumnIndexSuite.scala │ │ ├── RapidsParquetCompressionCodecPrecedenceSuite.scala │ │ ├── RapidsParquetDeltaByteArrayEncodingSuite.scala │ │ ├── RapidsParquetDeltaEncodingSuite.scala │ │ ├── RapidsParquetDeltaLengthByteArrayEncodingSuite.scala │ │ ├── RapidsParquetFieldIdIOSuite.scala │ │ ├── RapidsParquetFieldIdSchemaSuite.scala │ │ ├── RapidsParquetFileFormatSuite.scala │ │ ├── RapidsParquetInteroperabilitySuite.scala │ │ ├── RapidsParquetPartitionDiscoverySuite.scala │ │ ├── RapidsParquetProtobufCompatibilitySuite.scala │ │ ├── RapidsParquetQuerySuite.scala │ │ ├── RapidsParquetRebaseDatetimeSuite.scala │ │ ├── RapidsParquetSchemaPruningSuite.scala │ │ ├── RapidsParquetSchemaSuite.scala │ │ ├── RapidsParquetThriftCompatibilitySuite.scala │ │ ├── RapidsParquetVectorizedSuite.scala │ │ ├── RapidsRegexpExpressionsSuite.scala │ │ ├── RapidsStringExpressionsSuite.scala │ │ └── RapidsStringFunctionsSuite.scala │ │ └── utils │ │ ├── BackendTestSettings.scala │ │ ├── RapidsJsonConfTrait.scala │ │ ├── RapidsSQLTestsBaseTrait.scala │ │ ├── RapidsSQLTestsTrait.scala │ │ ├── RapidsTestConstants.scala │ │ ├── RapidsTestSettings.scala │ │ ├── RapidsTestsBaseTrait.scala │ │ ├── RapidsTestsCommonTrait.scala │ │ └── RapidsTestsTrait.scala │ ├── spark340 │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── shuffle │ │ └── RapidsShuffleTestHelper.scala │ ├── spark341db │ └── scala │ │ └── com │ │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── ToPrettyStringSuite.scala │ └── spark350 │ └── scala │ ├── com │ └── nvidia │ │ └── spark │ │ └── rapids │ │ └── iceberg │ │ ├── data │ │ └── GpuDeleteFilterSuite.scala │ │ └── datagen.scala │ └── org │ └── apache │ └── spark │ └── sql │ └── rapids │ └── GpuCreateDataSourceTableAsSelectCommandSuite.scala ├── tools ├── .gitignore ├── generated_files │ ├── 320 │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 321 │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 322 │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 323 │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 324 │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 330 │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 331 │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 332 │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 333 │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 334 │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 340 │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 341 │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 342 │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 343 │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 344 │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 350 │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 351 │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 352 │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 353 │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 354 │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 355 │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 356 │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 400 │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 321cdh │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 330cdh │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 332cdh │ │ ├── operatorsScore.csv │ │ ├── supportedDataSource.csv │ │ ├── supportedExecs.csv │ │ └── supportedExprs.csv │ ├── 350db143 │ │ └── supportedDataSource.csv │ ├── operatorsScore.csv │ ├── supportedDataSource.csv │ ├── supportedExecs.csv │ └── supportedExprs.csv └── pom.xml └── udf-compiler ├── README.md ├── pom.xml └── src ├── main ├── scala-2.12 │ └── com │ │ └── nvidia │ │ └── spark │ │ └── udf │ │ └── Instruction.scala ├── scala-2.13 │ └── com │ │ └── nvidia │ │ └── spark │ │ └── udf │ │ └── Instruction.scala └── scala │ └── com │ └── nvidia │ └── spark │ └── udf │ ├── CFG.scala │ ├── CatalystExpressionBuilder.scala │ ├── GpuScalaUDF.scala │ ├── LambdaReflection.scala │ ├── LogicalPlanRules.scala │ └── State.scala └── test ├── resources ├── log4j.properties └── log4j2.properties └── scala └── com └── nvidia └── spark └── OpcodeSuite.scala /.github/404.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | permalink: /404.html 4 | --- 5 | 6 | 19 | 20 |
21 |

404

22 | 23 |

Page not found :(

24 |

The requested page could not be found. 25 | If you are looking for documentation please navigate to the User Guide for more information

26 |
-------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020-2025, NVIDIA CORPORATION. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # Unless required by applicable law or agreed to in writing, software 7 | # distributed under the License is distributed on an "AS IS" BASIS, 8 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | # See the License for the specific language governing permissions and 10 | # limitations under the License. 11 | 12 | # Build-related 13 | /jenkins/ @NVIDIA/sparkrapids-cicd-codeowners 14 | pom.xml @NVIDIA/sparkrapids-cicd-codeowners 15 | /dist/ @NVIDIA/sparkrapids-cicd-codeowners 16 | /.github/ @NVIDIA/sparkrapids-cicd-codeowners 17 | /build/ @NVIDIA/sparkrapids-cicd-codeowners 18 | /scripts/ @NVIDIA/sparkrapids-cicd-codeowners 19 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a bug report to help us improve RAPIDS Accelerator for Apache Spark 4 | title: "[BUG]" 5 | labels: "? - Needs Triage, bug" 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **Steps/Code to reproduce bug** 14 | Please provide a list of steps or a code sample to reproduce the issue. 15 | Avoid posting private or sensitive data. 16 | 17 | **Expected behavior** 18 | A clear and concise description of what you expected to happen. 19 | 20 | **Environment details (please complete the following information)** 21 | - Environment location: [Standalone, YARN, Kubernetes, Cloud(specify cloud provider)] 22 | - Spark configuration settings related to the issue 23 | 24 | **Additional context** 25 | Add any other context about the problem here. 26 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/documentation-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Documentation request 3 | about: Report incorrect or needed documentation 4 | title: "[DOC]" 5 | labels: "? - Needs Triage, documentation" 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Report incorrect documentation 11 | 12 | **Location of incorrect documentation** 13 | Provide links and line numbers if applicable. 14 | 15 | **Describe the problems or issues found in the documentation** 16 | A clear and concise description of what you found to be incorrect. 17 | 18 | **Steps taken to verify documentation is incorrect** 19 | List any steps you have taken: 20 | 21 | **Suggested fix for documentation** 22 | Detail proposed changes to fix the documentation if you have any. 23 | 24 | --- 25 | 26 | ## Report needed documentation 27 | 28 | **Report needed documentation** 29 | A clear and concise description of what documentation you believe it is needed and why. 30 | 31 | **Describe the documentation you'd like** 32 | A clear and concise description of what you want to happen. 33 | 34 | **Steps taken to search for needed documentation** 35 | List any steps you have taken: 36 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for RAPIDS Accelerator for Apache Spark 4 | title: "[FEA]" 5 | labels: "? - Needs Triage, feature request" 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I wish the RAPIDS Accelerator for Apache Spark would [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context, code examples, or references to existing implementations about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/submit-question.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Submit question 3 | about: Ask a general question about RAPIDS Accelerator for Apache Spark here, or open a thread in the Discussions tab 4 | title: "[QST]" 5 | labels: "? - Needs Triage, question" 6 | assignees: '' 7 | 8 | --- 9 | 10 | **What is your question?** 11 | -------------------------------------------------------------------------------- /.github/codecov.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, NVIDIA CORPORATION. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | codecov: 16 | max_report_age: off 17 | -------------------------------------------------------------------------------- /.github/workflows/add-to-project.yml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022-2025, NVIDIA CORPORATION. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | name: Add new issues and pull requests to project 16 | 17 | on: 18 | issues: 19 | types: 20 | - opened 21 | pull_request_target: 22 | types: 23 | - opened 24 | 25 | jobs: 26 | Add-to-project: 27 | if: github.repository_owner == 'NVIDIA' # avoid adding issues from forks 28 | runs-on: ubuntu-latest 29 | steps: 30 | - name: add-to-project 31 | uses: NVIDIA/spark-rapids-common/add-to-project@main 32 | with: 33 | token: ${{ secrets.PROJECT_TOKEN }} 34 | -------------------------------------------------------------------------------- /.github/workflows/markdown-links-check/markdown-links-check-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "ignorePatterns": [ 3 | { 4 | "pattern": "https://github.com/NVIDIA/spark-rapids/pull/*" 5 | }, 6 | { 7 | "pattern": "https://github.com/NVIDIA/spark-rapids/issues/*" 8 | }, 9 | { 10 | "pattern": "/docs/archives" 11 | }, 12 | { 13 | "pattern": "http://localhost*" 14 | }, 15 | { 16 | "pattern": "https://www.nvidia.com/en-us/security/pgp-key" 17 | }, 18 | { 19 | "pattern": "https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.*" 20 | } 21 | ], 22 | "timeout": "15s", 23 | "retryOn429": true, 24 | "retryCount":30, 25 | "aliveStatusCodes": [200, 403] 26 | } 27 | -------------------------------------------------------------------------------- /.github/workflows/signoff-check.yml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020-2024, NVIDIA CORPORATION. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # A workflow to check if PR got sign-off 16 | name: signoff check 17 | 18 | on: 19 | pull_request_target: 20 | types: [opened, synchronize, reopened] 21 | 22 | jobs: 23 | signoff-check: 24 | runs-on: ubuntu-latest 25 | steps: 26 | - name: signoff 27 | uses: NVIDIA/spark-rapids-common/signoff-check@main 28 | with: 29 | owner: ${{ github.repository_owner }} 30 | repo: spark-rapids 31 | pull_number: ${{ github.event.number }} 32 | token: ${{ secrets.GITHUB_TOKEN }} 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # keep the lines below sorted 2 | 3 | # standard .bloop and version-specific .bloop3xy generated by buildall -gb 4 | .bloop* 5 | .cache 6 | .classpath 7 | .DS_Store 8 | .ensime 9 | .ensime_cache/ 10 | .ensime_lucene 11 | .generated-mima* 12 | .idea_modules/ 13 | .idea/ 14 | .mvn/ 15 | .metals/ 16 | .project 17 | .pydevproject 18 | .scala_dependencies 19 | .settings 20 | .vscode 21 | *.#* 22 | *.iml 23 | *.ipr 24 | *.iws 25 | *.pyc 26 | *.pyo 27 | *.swp 28 | */metastore_db 29 | */spark-warehouse 30 | *#*# 31 | *~ 32 | dependency-reduced-pom.xml 33 | hs_err*.log 34 | scalastyle-on-compile.generated.xml 35 | scalastyle-output.xml 36 | scalastyle.txt 37 | target/ 38 | cufile.log 39 | cudf_log.txt 40 | build/*.class 41 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "thirdparty/parquet-testing"] 2 | path = thirdparty/parquet-testing 3 | url = https://github.com/apache/parquet-testing 4 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, NVIDIA CORPORATION. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | repos: 16 | - repo: local 17 | hooks: 18 | - id: auto-copyrighter 19 | name: Update copyright year 20 | entry: scripts/auto-copyrighter.sh 21 | language: script 22 | pass_filenames: true 23 | verbose: true 24 | - repo: https://github.com/pre-commit/pre-commit-hooks.git 25 | rev: v4.0.1 26 | hooks: 27 | - id: check-added-large-files 28 | name: Check for file over 1.5MiB 29 | args: ['--maxkb=1500', '--enforce-all'] 30 | 31 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | ## Security 2 | 3 | NVIDIA is dedicated to the security and trust of our software products and services, including all 4 | source code repositories managed through our organization. 5 | 6 | If you need to report a security issue, please use the appropriate contact points outlined 7 | below. **Please do not report security vulnerabilities through GitHub/GitLab.** 8 | 9 | ## Reporting Potential Security Vulnerability in an NVIDIA Product 10 | 11 | To report a potential security vulnerability in any NVIDIA product: 12 | - Web: [Security Vulnerability Submission Form](https://www.nvidia.com/object/submit-security-vulnerability.html) 13 | - E-Mail: psirt@nvidia.com 14 | - We encourage you to use the following PGP key for secure email communication: [NVIDIA public PGP Key for communication](https://www.nvidia.com/en-us/security/pgp-key) 15 | - Please include the following information: 16 | - Product/Driver name and version/branch that contains the vulnerability 17 | -------------------------------------------------------------------------------- /api_validation/auditAllVersions.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2020-2024, NVIDIA CORPORATION. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -ex 16 | 17 | mvn scala:run -P spark320 18 | -------------------------------------------------------------------------------- /api_validation/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2020, NVIDIA CORPORATION. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | log4j.rootCategory=INFO, file 18 | log4j.appender.file=org.apache.log4j.FileAppender 19 | log4j.appender.file.append=true 20 | log4j.appender.file.file=target/api-validation-result.log 21 | log4j.appender.file.layout=org.apache.log4j.PatternLayout -------------------------------------------------------------------------------- /build/build-info: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # Copyright (c) 2019-2024, NVIDIA CORPORATION. All rights reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | # This script generates the build info. 20 | # Arguments: 21 | # rapids4spark_version - The current version of spark plugin 22 | set -e 23 | 24 | echo_build_properties() { 25 | echo version=$1 26 | echo cudf_version=$2 27 | echo user=$(whoami) 28 | echo revision=$(git rev-parse HEAD) 29 | echo branch=$(git rev-parse --abbrev-ref HEAD) 30 | echo date=$(date -u +%Y-%m-%dT%H:%M:%SZ) 31 | echo url=$(git config --get remote.origin.url) 32 | } 33 | 34 | echo_build_properties $1 $2 35 | -------------------------------------------------------------------------------- /datagen/src/main/spark400/scala/org/apache/spark/sql/tests/datagen/DataGenExprShims.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024-2025, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "400"} 19 | spark-rapids-shim-json-lines ***/ 20 | package org.apache.spark.sql.tests.datagen 21 | 22 | import org.apache.spark.sql.Column 23 | import org.apache.spark.sql.catalyst.expressions.Expression 24 | import org.apache.spark.sql.classic.ExpressionUtils.{column, expression} 25 | 26 | object DataGenExprShims { 27 | def columnToExpr(c: Column): Expression = expression(c) 28 | def exprToColumn(e: Expression): Column = column(e) 29 | } 30 | -------------------------------------------------------------------------------- /delta-lake/common/src/main/delta-20x-24x/scala/org/apache/spark/sql/delta/shims/ShimOptimisticTransaction.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.apache.spark.sql.delta.rapids 18 | 19 | import org.apache.spark.sql.catalyst.catalog.CatalogTable 20 | import org.apache.spark.sql.delta.{DeltaLog, OptimisticTransaction, Snapshot} 21 | import org.apache.spark.util.Clock 22 | 23 | class ShimOptimisticTransaction( 24 | deltaLog: DeltaLog, 25 | catalog: Option[CatalogTable], 26 | snapshot: Snapshot) 27 | (implicit clock: Clock) 28 | extends OptimisticTransaction(deltaLog, snapshot)(clock) 29 | -------------------------------------------------------------------------------- /delta-lake/common/src/main/delta-33x/scala/org/apache/spark/sql/delta/shims/ShimOptimisticTransaction.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.apache.spark.sql.delta.rapids 18 | 19 | import org.apache.spark.sql.catalyst.catalog.CatalogTable 20 | import org.apache.spark.sql.delta.{DeltaLog, OptimisticTransaction, Snapshot} 21 | import org.apache.spark.util.Clock 22 | 23 | class ShimOptimisticTransaction( 24 | deltaLog: DeltaLog, 25 | catalog: Option[CatalogTable], 26 | snapshot: Snapshot) 27 | (implicit clock: Clock) 28 | extends OptimisticTransaction(deltaLog, catalog, snapshot) 29 | -------------------------------------------------------------------------------- /delta-lake/common/src/main/delta-io/scala/com/nvidia/spark/rapids/delta/shims/ShimDeltaUDF.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.nvidia.spark.rapids.delta.shims 18 | 19 | import org.apache.spark.sql.delta.rapids.DeltaRuntimeShim 20 | import org.apache.spark.sql.expressions.UserDefinedFunction 21 | 22 | object ShimDeltaUDF { 23 | def stringStringUdf(f: String => String): UserDefinedFunction = { 24 | DeltaRuntimeShim.stringFromStringUdf(f) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /delta-lake/delta-20x/src/main/scala/com/nvidia/spark/rapids/delta/delta20x/GpuDelta20xParquetFileFormat.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.nvidia.spark.rapids.delta.delta20x 18 | 19 | import com.nvidia.spark.rapids.delta.GpuDeltaParquetFileFormat 20 | 21 | import org.apache.spark.sql.delta.DeltaColumnMappingMode 22 | import org.apache.spark.sql.types.StructType 23 | 24 | case class GpuDelta20xParquetFileFormat( 25 | override val columnMappingMode: DeltaColumnMappingMode, 26 | override val referenceSchema: StructType) extends GpuDeltaParquetFileFormat { 27 | } 28 | -------------------------------------------------------------------------------- /delta-lake/delta-20x/src/main/scala/com/nvidia/spark/rapids/delta/shims/MetadataShims.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.nvidia.spark.rapids.delta.shims 18 | 19 | import org.apache.spark.sql.delta.stats.UsesMetadataFields 20 | 21 | trait ShimUsesMetadataFields extends UsesMetadataFields -------------------------------------------------------------------------------- /delta-lake/delta-21x/src/main/scala/com/nvidia/spark/rapids/delta/delta21x/GpuDelta21xParquetFileFormat.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.nvidia.spark.rapids.delta.delta21x 18 | 19 | import com.nvidia.spark.rapids.delta.GpuDeltaParquetFileFormat 20 | 21 | import org.apache.spark.sql.delta.DeltaColumnMappingMode 22 | import org.apache.spark.sql.types.StructType 23 | 24 | case class GpuDelta21xParquetFileFormat( 25 | override val columnMappingMode: DeltaColumnMappingMode, 26 | override val referenceSchema: StructType) extends GpuDeltaParquetFileFormat { 27 | } 28 | -------------------------------------------------------------------------------- /delta-lake/delta-21x/src/main/scala/com/nvidia/spark/rapids/delta/shims/MetadataShims.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.nvidia.spark.rapids.delta.shims 17 | 18 | import org.apache.spark.sql.delta.stats.UsesMetadataFields 19 | 20 | trait ShimUsesMetadataFields extends UsesMetadataFields 21 | -------------------------------------------------------------------------------- /delta-lake/delta-22x/src/main/scala/com/nvidia/spark/rapids/delta/shims/MetadataShims.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.nvidia.spark.rapids.delta.shims 17 | 18 | import org.apache.spark.sql.delta.stats.UsesMetadataFields 19 | 20 | trait ShimUsesMetadataFields extends UsesMetadataFields -------------------------------------------------------------------------------- /delta-lake/delta-23x/src/main/scala/com/nvidia/spark/rapids/delta/shims/MetadataShims.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.nvidia.spark.rapids.delta.shims 17 | 18 | import org.apache.spark.sql.delta.stats.UsesMetadataFields 19 | 20 | trait ShimUsesMetadataFields extends UsesMetadataFields -------------------------------------------------------------------------------- /delta-lake/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/shims/MetadataShims.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.nvidia.spark.rapids.delta.shims 17 | 18 | import org.apache.spark.sql.delta.stats.DeltaStatistics 19 | 20 | trait ShimUsesMetadataFields { 21 | val NUM_RECORDS = DeltaStatistics.NUM_RECORDS 22 | val MIN = DeltaStatistics.MIN 23 | val MAX = DeltaStatistics.MAX 24 | val NULL_COUNT = DeltaStatistics.NULL_COUNT 25 | } -------------------------------------------------------------------------------- /delta-lake/delta-33x/src/main/scala/com/nvidia/spark/rapids/delta/shims/MetadataShims.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.nvidia.spark.rapids.delta.shims 17 | 18 | import org.apache.spark.sql.delta.stats.DeltaStatistics 19 | 20 | trait ShimUsesMetadataFields { 21 | protected val MAX = DeltaStatistics.MAX 22 | protected val MIN = DeltaStatistics.MIN 23 | protected val NUM_RECORDS = DeltaStatistics.NUM_RECORDS 24 | protected val NULL_COUNT = DeltaStatistics.NULL_COUNT 25 | } 26 | -------------------------------------------------------------------------------- /delta-lake/delta-spark330db/src/main/scala/com/nvidia/spark/rapids/delta/DeltaProbe.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.nvidia.spark.rapids.delta 17 | 18 | /** 19 | * Implements the Delta Probe interface for probing the Delta Lake provider on Databricks. 20 | * @note This is instantiated via reflection from ShimLoader. 21 | */ 22 | class DeltaProbeImpl extends DeltaProbe { 23 | // Delta Lake is built-in for Databricks instances, so no probing is necessary. 24 | override def getDeltaProvider: DeltaProvider = DeltaSpark330DBProvider 25 | } 26 | -------------------------------------------------------------------------------- /delta-lake/delta-spark330db/src/main/scala/com/nvidia/spark/rapids/delta/shims/DeleteCommandMetaShim.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.nvidia.spark.rapids.delta.shims 18 | 19 | import com.nvidia.spark.rapids.delta.{DeleteCommandEdgeMeta, DeleteCommandMeta} 20 | 21 | object DeleteCommandMetaShim { 22 | def tagForGpu(meta: DeleteCommandMeta): Unit = {} 23 | def tagForGpu(meta: DeleteCommandEdgeMeta): Unit = {} 24 | } -------------------------------------------------------------------------------- /delta-lake/delta-spark330db/src/main/scala/com/nvidia/spark/rapids/delta/shims/MetadataShims.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.nvidia.spark.rapids.delta.shims 17 | 18 | import com.databricks.sql.transaction.tahoe.stats.UsesMetadataFields 19 | 20 | trait ShimUsesMetadataFields extends UsesMetadataFields 21 | -------------------------------------------------------------------------------- /delta-lake/delta-spark330db/src/main/scala/com/nvidia/spark/rapids/delta/shims/ShimDeltaUDF.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.nvidia.spark.rapids.delta.shims 18 | 19 | import com.databricks.sql.transaction.tahoe.DeltaUDF 20 | 21 | import org.apache.spark.sql.expressions.UserDefinedFunction 22 | 23 | object ShimDeltaUDF { 24 | def stringStringUdf(f: String => String): UserDefinedFunction = DeltaUDF.stringFromString(f) 25 | } 26 | -------------------------------------------------------------------------------- /delta-lake/delta-spark330db/src/main/scala/com/nvidia/spark/rapids/delta/shims/UpdateCommandMetaShim.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.nvidia.spark.rapids.delta.shims 18 | 19 | import com.nvidia.spark.rapids.delta.{UpdateCommandEdgeMeta, UpdateCommandMeta} 20 | 21 | object UpdateCommandMetaShim { 22 | def tagForGpu(meta: UpdateCommandMeta): Unit = {} 23 | 24 | def tagForGpu(meta: UpdateCommandEdgeMeta): Unit = {} 25 | } 26 | -------------------------------------------------------------------------------- /delta-lake/delta-spark332db/src/main/scala/com/nvidia/spark/rapids/delta/DeltaProbe.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.nvidia.spark.rapids.delta 17 | 18 | /** 19 | * Implements the Delta Probe interface for probing the Delta Lake provider on Databricks. 20 | * @note This is instantiated via reflection from ShimLoader. 21 | */ 22 | class DeltaProbeImpl extends DeltaProbe { 23 | // Delta Lake is built-in for Databricks instances, so no probing is necessary. 24 | override def getDeltaProvider: DeltaProvider = DeltaSpark332DBProvider 25 | } 26 | -------------------------------------------------------------------------------- /delta-lake/delta-spark332db/src/main/scala/com/nvidia/spark/rapids/delta/shims/MetadataShims.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.nvidia.spark.rapids.delta.shims 17 | 18 | import com.databricks.sql.transaction.tahoe.stats.UsesMetadataFields 19 | 20 | trait ShimUsesMetadataFields extends UsesMetadataFields 21 | -------------------------------------------------------------------------------- /delta-lake/delta-spark332db/src/main/scala/com/nvidia/spark/rapids/delta/shims/ShimDeltaUDF.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.nvidia.spark.rapids.delta.shims 18 | 19 | import com.databricks.sql.transaction.tahoe.DeltaUDF 20 | 21 | import org.apache.spark.sql.expressions.UserDefinedFunction 22 | 23 | object ShimDeltaUDF { 24 | def stringStringUdf(f: String => String): UserDefinedFunction = DeltaUDF.stringFromString(f) 25 | } 26 | -------------------------------------------------------------------------------- /delta-lake/delta-spark332db/src/main/scala/com/nvidia/spark/rapids/delta/shims/UpdateCommandMetaShim.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.nvidia.spark.rapids.delta.shims 18 | 19 | import com.nvidia.spark.rapids.delta.{UpdateCommandEdgeMeta, UpdateCommandMeta} 20 | 21 | object UpdateCommandMetaShim { 22 | def tagForGpu(meta: UpdateCommandMeta): Unit = {} 23 | 24 | def tagForGpu(meta: UpdateCommandEdgeMeta): Unit = {} 25 | } 26 | -------------------------------------------------------------------------------- /delta-lake/delta-spark341db/src/main/scala/com/nvidia/spark/rapids/delta/DeltaProbe.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.nvidia.spark.rapids.delta 17 | 18 | /** 19 | * Implements the Delta Probe interface for probing the Delta Lake provider on Databricks. 20 | * @note This is instantiated via reflection from ShimLoader. 21 | */ 22 | class DeltaProbeImpl extends DeltaProbe { 23 | // Delta Lake is built-in for Databricks instances, so no probing is necessary. 24 | override def getDeltaProvider: DeltaProvider = DeltaSpark341DBProvider 25 | } 26 | -------------------------------------------------------------------------------- /delta-lake/delta-spark341db/src/main/scala/com/nvidia/spark/rapids/delta/shims/MetadataShims.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.nvidia.spark.rapids.delta.shims 17 | 18 | import com.databricks.sql.transaction.tahoe.stats.DeltaStatistics 19 | 20 | trait ShimUsesMetadataFields { 21 | val NUM_RECORDS = DeltaStatistics.NUM_RECORDS 22 | val MIN = DeltaStatistics.MIN 23 | val MAX = DeltaStatistics.MAX 24 | val NULL_COUNT = DeltaStatistics.NULL_COUNT 25 | } -------------------------------------------------------------------------------- /delta-lake/delta-spark341db/src/main/scala/com/nvidia/spark/rapids/delta/shims/ShimDeltaUDF.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.nvidia.spark.rapids.delta.shims 18 | 19 | import com.databricks.sql.transaction.tahoe.DeltaUDF 20 | 21 | import org.apache.spark.sql.expressions.UserDefinedFunction 22 | 23 | object ShimDeltaUDF { 24 | def stringStringUdf(f: String => String): UserDefinedFunction = DeltaUDF.stringFromString(f) 25 | } 26 | -------------------------------------------------------------------------------- /delta-lake/delta-spark341db/src/main/scala/com/nvidia/spark/rapids/delta/shims/UpdateCommandMetaShim.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.nvidia.spark.rapids.delta.shims 18 | 19 | import com.nvidia.spark.rapids.delta.{UpdateCommandEdgeMeta, UpdateCommandMeta} 20 | 21 | object UpdateCommandMetaShim { 22 | def tagForGpu(meta: UpdateCommandMeta): Unit = {} 23 | 24 | def tagForGpu(meta: UpdateCommandEdgeMeta): Unit = {} 25 | } 26 | -------------------------------------------------------------------------------- /delta-lake/delta-spark350db143/src/main/scala/com/nvidia/spark/rapids/delta/DeltaProbe.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024-2025, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.nvidia.spark.rapids.delta 17 | 18 | /** 19 | * Implements the Delta Probe interface for probing the Delta Lake provider on Databricks. 20 | * @note This is instantiated via reflection from ShimLoader. 21 | */ 22 | class DeltaProbeImpl extends DeltaProbe { 23 | // Delta Lake is built-in for Databricks instances, so no probing is necessary. 24 | override def getDeltaProvider: DeltaProvider = DeltaSpark350DB143Provider 25 | } 26 | -------------------------------------------------------------------------------- /delta-lake/delta-spark350db143/src/main/scala/com/nvidia/spark/rapids/delta/shims/MetadataShims.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.nvidia.spark.rapids.delta.shims 17 | 18 | import com.databricks.sql.transaction.tahoe.stats.DeltaStatistics 19 | 20 | trait ShimUsesMetadataFields { 21 | val NUM_RECORDS = DeltaStatistics.NUM_RECORDS 22 | val MIN = DeltaStatistics.MIN 23 | val MAX = DeltaStatistics.MAX 24 | val NULL_COUNT = DeltaStatistics.NULL_COUNT 25 | } 26 | -------------------------------------------------------------------------------- /delta-lake/delta-spark350db143/src/main/scala/com/nvidia/spark/rapids/delta/shims/ShimDeltaUDF.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.nvidia.spark.rapids.delta.shims 18 | 19 | import com.databricks.sql.transaction.tahoe.DeltaUDF 20 | 21 | import org.apache.spark.sql.expressions.UserDefinedFunction 22 | 23 | object ShimDeltaUDF { 24 | def stringStringUdf(f: String => String): UserDefinedFunction = DeltaUDF.stringFromString(f) 25 | } 26 | -------------------------------------------------------------------------------- /delta-lake/delta-stub/src/main/scala/com/nvidia/spark/rapids/delta/DeltaProbeImpl.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.nvidia.spark.rapids.delta 18 | 19 | /** 20 | * Implements the Delta Probe interface for platforms where Delta Lake is not supported. 21 | * @note This is instantiated via reflection from ShimLoader 22 | */ 23 | class DeltaProbeImpl extends DeltaProbe { 24 | override def getDeltaProvider: DeltaProvider = NoDeltaProvider 25 | } 26 | -------------------------------------------------------------------------------- /dist/unshimmed-common-from-spark320.txt: -------------------------------------------------------------------------------- 1 | META-INF/DEPENDENCIES 2 | META-INF/LICENSE 3 | META-INF/NOTICE 4 | com/nvidia/spark/GpuCachedBatchSerializer* 5 | com/nvidia/spark/ParquetCachedBatchSerializer* 6 | com/nvidia/spark/rapids/ExplainPlan.class 7 | com/nvidia/spark/rapids/ExplainPlan$.class 8 | com/nvidia/spark/rapids/ExplainPlanBase.class 9 | com/nvidia/spark/rapids/optimizer/SQLOptimizerPlugin* 10 | com/nvidia/spark/rapids/ShimLoaderTemp* 11 | com/nvidia/spark/rapids/SparkShims* 12 | org/apache/spark/sql/rapids/AdaptiveSparkPlanHelperShim* 13 | org/apache/spark/sql/rapids/ExecutionPlanCaptureCallback* 14 | rapids/*.py 15 | -------------------------------------------------------------------------------- /dist/unshimmed-from-each-spark3xx.txt: -------------------------------------------------------------------------------- 1 | com/nvidia/spark/rapids/*/RapidsShuffleManager.class 2 | com/nvidia/spark/rapids/AvroProvider.class 3 | com/nvidia/spark/rapids/HiveProvider.class 4 | com/nvidia/spark/rapids/iceberg/IcebergProvider.class 5 | com/nvidia/spark/rapids/delta/DeltaProbe.class 6 | com/nvidia/spark/rapids/delta/DeltaProvider.class 7 | com/nvidia/spark/rapids/delta/DeltaProvider$.class 8 | com/nvidia/spark/rapids/PlanShims* 9 | spark-*-info.properties 10 | -------------------------------------------------------------------------------- /docs/_includes/analytics.html: -------------------------------------------------------------------------------- 1 | 18 | 19 | 20 | 21 | 22 | 29 | 30 | -------------------------------------------------------------------------------- /docs/additional-functionality/README.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: Additional Functionality 4 | nav_order: 10 5 | has_children: true 6 | permalink: /additional-functionality/ 7 | --- 8 | -------------------------------------------------------------------------------- /docs/contact-us.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: Contact Us 4 | nav_order: 16 5 | --- 6 | # Contact Us 7 | 8 | ## Technical Support 9 | If you need any help or have questions on this product, please contact us: 10 | spark-rapids-support@nvidia.com 11 | 12 | ## Issues and Discussions 13 | We use github to track bugs, feature requests, and answer questions. File an 14 | [issue](https://github.com/NVIDIA/spark-rapids/issues/new/choose) for a bug or feature request. Ask 15 | or answer a question on the [discussion board](https://github.com/NVIDIA/spark-rapids/discussions). -------------------------------------------------------------------------------- /docs/dev/testing.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: Testing 4 | nav_order: 2 5 | parent: Developer Overview 6 | --- 7 | An overview of testing can be found within the repository at: 8 | * [Unit tests](https://github.com/NVIDIA/spark-rapids/tree/branch-25.08/tests#readme) 9 | * [Integration testing](https://github.com/NVIDIA/spark-rapids/tree/branch-25.08/integration_tests#readme) 10 | -------------------------------------------------------------------------------- /docs/img/AWS-EMR/EMR_notebook_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/AWS-EMR/EMR_notebook_2.png -------------------------------------------------------------------------------- /docs/img/AWS-EMR/EMR_notebook_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/AWS-EMR/EMR_notebook_3.png -------------------------------------------------------------------------------- /docs/img/AWS-EMR/bootstrap-action.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/AWS-EMR/bootstrap-action.png -------------------------------------------------------------------------------- /docs/img/AWS-EMR/cluster-configuration.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/AWS-EMR/cluster-configuration.png -------------------------------------------------------------------------------- /docs/img/AWS-EMR/name-and-applications.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/AWS-EMR/name-and-applications.png -------------------------------------------------------------------------------- /docs/img/AWS-EMR/networking.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/AWS-EMR/networking.png -------------------------------------------------------------------------------- /docs/img/AWS-EMR/notebook-workspace-creation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/AWS-EMR/notebook-workspace-creation.png -------------------------------------------------------------------------------- /docs/img/AWS-EMR/spark-history-server-UI.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/AWS-EMR/spark-history-server-UI.png -------------------------------------------------------------------------------- /docs/img/AWS-EMR/ssh-key-pair.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/AWS-EMR/ssh-key-pair.png -------------------------------------------------------------------------------- /docs/img/Databricks/a-metric-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/Databricks/a-metric-graph.png -------------------------------------------------------------------------------- /docs/img/Databricks/grafana.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/Databricks/grafana.png -------------------------------------------------------------------------------- /docs/img/Databricks/initscript.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/Databricks/initscript.png -------------------------------------------------------------------------------- /docs/img/Databricks/initscript_ucx.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/Databricks/initscript_ucx.png -------------------------------------------------------------------------------- /docs/img/Databricks/list-metrics-result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/Databricks/list-metrics-result.png -------------------------------------------------------------------------------- /docs/img/Databricks/list-metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/Databricks/list-metrics.png -------------------------------------------------------------------------------- /docs/img/Databricks/prometheus-datasource.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/Databricks/prometheus-datasource.png -------------------------------------------------------------------------------- /docs/img/Databricks/save-prometheus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/Databricks/save-prometheus.png -------------------------------------------------------------------------------- /docs/img/Databricks/sparkconfig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/Databricks/sparkconfig.png -------------------------------------------------------------------------------- /docs/img/Databricks/sparkconfig_ucx.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/Databricks/sparkconfig_ucx.png -------------------------------------------------------------------------------- /docs/img/GCP/dataproc-cluster.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/GCP/dataproc-cluster.png -------------------------------------------------------------------------------- /docs/img/GCP/dataproc-service.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/GCP/dataproc-service.png -------------------------------------------------------------------------------- /docs/img/Tools/qualification-tool-app-view-01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/Tools/qualification-tool-app-view-01.png -------------------------------------------------------------------------------- /docs/img/Tools/qualification-tool-app-view-02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/Tools/qualification-tool-app-view-02.png -------------------------------------------------------------------------------- /docs/img/Tools/qualification-tool-app-view-03.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/Tools/qualification-tool-app-view-03.png -------------------------------------------------------------------------------- /docs/img/Tools/qualification-tool-recommendation-indexview-with-persql.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/Tools/qualification-tool-recommendation-indexview-with-persql.png -------------------------------------------------------------------------------- /docs/img/concurrentPythonWorker.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/concurrentPythonWorker.PNG -------------------------------------------------------------------------------- /docs/img/join-sql-ui-example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/join-sql-ui-example.png -------------------------------------------------------------------------------- /docs/img/perf-cost.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/perf-cost.png -------------------------------------------------------------------------------- /docs/img/spark3cluster.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/spark3cluster.png -------------------------------------------------------------------------------- /docs/img/tpcxbb-like-results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/docs/img/tpcxbb-like-results.png -------------------------------------------------------------------------------- /docs/security.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: Security 4 | nav_order: 13 5 | --- 6 | ## Security 7 | 8 | NVIDIA is dedicated to the security and trust of our software products and services, including all 9 | source code repositories managed through our organization. 10 | 11 | If you need to report a security issue, please use the appropriate contact points outlined 12 | below. **Please do not report security vulnerabilities through GitHub/GitLab.** 13 | 14 | ## Reporting Potential Security Vulnerability in an NVIDIA Product 15 | 16 | To report a potential security vulnerability in any NVIDIA product: 17 | - Web: [Security Vulnerability Submission Form](https://www.nvidia.com/object/submit-security-vulnerability.html) 18 | - E-Mail: psirt@nvidia.com 19 | - We encourage you to use the following PGP key for secure email communication: [NVIDIA public PGP Key for communication](https://www.nvidia.com/en-us/security/pgp-key) 20 | - Please include the following information: 21 | - Product/Driver name and version/branch that contains the vulnerability 22 | -------------------------------------------------------------------------------- /iceberg/src/main/java/org/apache/iceberg/spark/source/GpuStructInternalRow.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.apache.iceberg.spark.source; 18 | 19 | import org.apache.iceberg.types.Types; 20 | 21 | public class GpuStructInternalRow extends StructInternalRow { 22 | public GpuStructInternalRow(Types.StructType type) { 23 | super(type); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /iceberg/src/main/scala/com/nvidia/spark/rapids/iceberg/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.nvidia.spark.rapids 18 | 19 | import org.apache.iceberg.Schema 20 | import scala.collection.JavaConverters._ 21 | 22 | package object iceberg { 23 | private[iceberg] def fieldIndex(schema: Schema, fieldId: Int): Int = { 24 | val idx = schema 25 | .columns() 26 | .asScala 27 | .indexWhere(_.fieldId() == fieldId) 28 | if (idx == -1) { 29 | throw new IllegalArgumentException(s"Field id $fieldId not found in schema") 30 | } else { 31 | idx 32 | } 33 | } 34 | } -------------------------------------------------------------------------------- /iceberg/src/main/scala/com/nvidia/spark/rapids/iceberg/spark/GpuSparkReadOptions.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | 18 | package com.nvidia.spark.rapids.iceberg.spark 19 | 20 | object GpuSparkReadOptions { 21 | // Controls whether to allow reading timestamps without zone info 22 | val HANDLE_TIMESTAMP_WITHOUT_TIMEZONE = "handle-timestamp-without-timezone" 23 | } 24 | -------------------------------------------------------------------------------- /iceberg/src/main/scala/org/apache/iceberg/parquet/GpuParquetIO.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.apache.iceberg.parquet 18 | 19 | import org.apache.iceberg.io.InputFile 20 | import org.apache.iceberg.shaded.org.apache.parquet.io.{InputFile => ShadedInputFile} 21 | 22 | object GpuParquetIO { 23 | def file(file: InputFile): ShadedInputFile = { 24 | ParquetIO.file(file) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /iceberg/src/main/scala/org/apache/iceberg/spark/GpuTypeToSparkType.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.apache.iceberg.spark 18 | 19 | import org.apache.iceberg.Schema 20 | import org.apache.iceberg.types.TypeUtil 21 | 22 | import org.apache.spark.sql.types.StructType 23 | 24 | object GpuTypeToSparkType { 25 | def toSparkType(schema: Schema): StructType = { 26 | TypeUtil.visit(schema, new TypeToSparkType).asInstanceOf[StructType] 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /integration_tests/requirements.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-2025, NVIDIA CORPORATION. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | pytest 15 | sre_yield 16 | pandas 17 | pyarrow == 17.0.0 ; python_version == '3.8' 18 | pyarrow == 19.0.1 ; python_version >= '3.9' 19 | pytest-xdist >= 2.0.0 20 | findspark 21 | fsspec == 2025.3.0 22 | fastparquet == 0.8.3 ; python_version == '3.8' 23 | fastparquet == 2024.5.0 ; python_version >= '3.9' 24 | -------------------------------------------------------------------------------- /integration_tests/runtests.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, NVIDIA CORPORATION. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import sys 16 | 17 | from pytest import main 18 | 19 | #import cProfile 20 | 21 | if __name__ == '__main__': 22 | #cProfile.run('main(sys.argv[1:])', 'test_profile') 23 | # arguments are the same as for pytest https://docs.pytest.org/en/latest/usage.html 24 | # or run pytest -h 25 | sys.exit(main(sys.argv[1:])) 26 | -------------------------------------------------------------------------------- /integration_tests/src/main/java/com/nvidia/spark/rapids/tests/udf/hive/EmptyHiveSimpleUDF.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021-2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.nvidia.spark.rapids.tests.udf.hive; 18 | 19 | /** An empty Hive simple UDF returning the first input directly for row-based UDF test only. */ 20 | @SuppressWarnings("deprecation") 21 | public class EmptyHiveSimpleUDF extends org.apache.hadoop.hive.ql.exec.UDF { 22 | public String evaluate(String in, String in2) { 23 | return in; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /integration_tests/src/main/python/mortgage_test.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, NVIDIA CORPORATION. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import pytest 16 | 17 | from asserts import assert_gpu_and_cpu_are_equal_iterator 18 | from marks import approximate_float, incompat, ignore_order, allow_non_gpu, limit 19 | 20 | @incompat 21 | @approximate_float 22 | @limit 23 | @ignore_order 24 | @allow_non_gpu(any=True) 25 | def test_mortgage(mortgage): 26 | assert_gpu_and_cpu_are_equal_iterator( 27 | lambda spark : mortgage.do_test_query(spark)) 28 | -------------------------------------------------------------------------------- /integration_tests/src/main/scala/com/nvidia/spark/rapids/tests/scaletest/Utils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.nvidia.spark.rapids.tests.scaletest 18 | 19 | import java.io.{PrintWriter, StringWriter} 20 | 21 | object Utils { 22 | def stackTraceAsString(e: Throwable): String = { 23 | val sw = new StringWriter() 24 | val w = new PrintWriter(sw) 25 | e.printStackTrace(w) 26 | w.close() 27 | sw.toString 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/SPARK-32639/000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/integration_tests/src/test/resources/SPARK-32639/000.snappy.parquet -------------------------------------------------------------------------------- /integration_tests/src/test/resources/V_0_11.orc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/integration_tests/src/test/resources/V_0_11.orc -------------------------------------------------------------------------------- /integration_tests/src/test/resources/V_0_12.orc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/integration_tests/src/test/resources/V_0_12.orc -------------------------------------------------------------------------------- /integration_tests/src/test/resources/bad_whitespace.json: -------------------------------------------------------------------------------- 1 | {"data": 1 . 0} 2 | {"data": - 1 . 0} 3 | {"data": + 1 . 0} 4 | {"data": 1 E 1} 5 | {"data": n u l l} 6 | {"data": t r u e} 7 | {"data": f a l s e} 8 | {"data": 1 0} 9 | {"data": 1, "other": 1 0} 10 | {"data": "BAD NUM 1 000", "ride-along-num": 1 000} 11 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/binary_as_string.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/integration_tests/src/test/resources/binary_as_string.parquet -------------------------------------------------------------------------------- /integration_tests/src/test/resources/boolean.json: -------------------------------------------------------------------------------- 1 | { "number": true } 2 | { "number": True } 3 | { "number": TRUE } 4 | { "number": false } 5 | { "number": False } 6 | { "number": FALSE } 7 | { "number": null } 8 | { "number": y } 9 | { "number": n } 10 | { "number": 0 } 11 | { "number": 1 } -------------------------------------------------------------------------------- /integration_tests/src/test/resources/boolean_formatted.json: -------------------------------------------------------------------------------- 1 | {"data": "true"} 2 | {"data": "false"} 3 | {"data": "null"} 4 | {"data": ""} 5 | {"data": "True"} 6 | {"data": "TRUE"} 7 | {"data": "False"} 8 | {"data": "FALSE"} 9 | {"data": "BAD"} 10 | {"data": true } 11 | {"data": True} 12 | {"data": TRUE} 13 | {"data": false} 14 | {"data": False} 15 | {"data": FALSE} 16 | {"data": null} 17 | {"data": y} 18 | {"data": n} 19 | {"data": 0} 20 | {"data": 1} 21 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/boolean_invalid.json: -------------------------------------------------------------------------------- 1 | { "number": "true" } 2 | { "number": "false" } 3 | { "number": "null" } 4 | { "number": "" } 5 | { "number": "True" } 6 | { "number": "TRUE" } 7 | { "number": "False" } 8 | { "number": "FALSE" } 9 | { "number": "BAD" } -------------------------------------------------------------------------------- /integration_tests/src/test/resources/date.csv: -------------------------------------------------------------------------------- 1 | 2020-09-16 2 | 2020-10-16 3 | 2021-09-16 4 | 2021-09-16 5 | 2021-09-16 6 | 2020-50-16 7 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/dates.json: -------------------------------------------------------------------------------- 1 | { 'number': '2020-09-16' } 2 | { 'number': ' 2020-09-16' } 3 | { 'number': '2020-09-16 ' } 4 | { 'number': '1581-01-01' } 5 | { 'number': '1583-01-01' } -------------------------------------------------------------------------------- /integration_tests/src/test/resources/dates_invalid.json: -------------------------------------------------------------------------------- 1 | { "number": "2020-09-32" } 2 | { "number": "2020-50-16" } -------------------------------------------------------------------------------- /integration_tests/src/test/resources/decimal_locale_formatted_strings.json: -------------------------------------------------------------------------------- 1 | {"data":"1,000.001"} 2 | {"data":"1.000,001"} 3 | {"data":"1 000,001"} 4 | {"data":"1'000.001"} 5 | {"data":"1000.001"} 6 | {"data":"1000"} 7 | {"data":"BAD"} 8 | {"data": "1,2,3,4"} 9 | {"data": "1,2,3.4"} 10 | {"data": "0123.4"} 11 | {"data": "0,1234"} 12 | {"data": " 1234"} 13 | {"data": "1234 "} 14 | {"data": "1 234"} 15 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/decimal_locale_formatted_strings_non_aribic.json: -------------------------------------------------------------------------------- 1 | {"data":"१,०००.००१"} 2 | {"data":"๑,๐๐๐.๐๐๑"} 3 | {"data": "१"} 4 | {"data": "๑"} 5 | {"data": "१०"} 6 | {"data": "๑๐"} 7 | {"data": "1٢७"} 8 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/decimals.csv: -------------------------------------------------------------------------------- 1 | "number" 2 | -1 3 | 0 4 | 1. 5 | 0.12 6 | .12 7 | +.12 8 | -.12 9 | 1 10 | 1.01 11 | 12.34 12 | 12.3456 13 | 12345678.12 14 | 33.545454 15 | 33.454545 16 | 17 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/decimals.json: -------------------------------------------------------------------------------- 1 | { "number": 0 } 2 | { "number": 12 } 3 | { "number": 12.0 } 4 | { "number": 12. } 5 | { "number": .34 } 6 | { "number": +.34 } 7 | { "number": -.34 } 8 | { "number": 0.34 } 9 | { "number": 12.34 } 10 | { "number": 12.3456 } 11 | { "number": 12.345678 } 12 | { "number": 123456.78 } 13 | { "number": 33.454545 } 14 | { "number": 33.545454 } 15 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/empty.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/integration_tests/src/test/resources/empty.csv -------------------------------------------------------------------------------- /integration_tests/src/test/resources/empty_arrow.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/integration_tests/src/test/resources/empty_arrow.parquet -------------------------------------------------------------------------------- /integration_tests/src/test/resources/empty_int_values.csv: -------------------------------------------------------------------------------- 1 | empty-0space,,end 2 | empty-0space-quoted,"",end 3 | empty-1space, ,end 4 | empty-1space-quoted," ",end 5 | empty-2space, ,end 6 | empty-2space-quoted," ",end 7 | no-null,3,end 8 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/fixed-length-char-column-from-hive.orc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/integration_tests/src/test/resources/fixed-length-char-column-from-hive.orc -------------------------------------------------------------------------------- /integration_tests/src/test/resources/float_formatted.json: -------------------------------------------------------------------------------- 1 | {"data": 1.0} 2 | {"data": 1.} 3 | {"data": -0.0} 4 | {"data": 0.0} 5 | {"data": "+INF"} 6 | {"data": "-INF"} 7 | {"data": +INF} 8 | {"data": -INF} 9 | {"data": "+Infinity"} 10 | {"data": "-Infinity"} 11 | {"data": +Infinity} 12 | {"data": -Infinity} 13 | {"data": NaN} 14 | {"data":"NaN"} 15 | {"data": 1.00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001} 16 | {"data": 99999999999999999999999999999999999999.00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001} 17 | {"data": 0.9} 18 | {"data": 0.99} 19 | {"data": 0.999} 20 | {"data": 0.9999} 21 | {"data": +1.0} 22 | {"data": -1.0} 23 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/float_formatted_strings.json: -------------------------------------------------------------------------------- 1 | {"data": "1.0"} 2 | {"data": "1."} 3 | {"data": "-0.0"} 4 | {"data": "0.0"} 5 | {"data": "+INF"} 6 | {"data": "-INF"} 7 | {"data": "+Infinity"} 8 | {"data": "-Infinity"} 9 | {"data": "NaN"} 10 | {"data": "1.00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"} 11 | {"data": "99999999999999999999999999999999999999.00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"} 12 | {"data": "0.9"} 13 | {"data": "0.99"} 14 | {"data": "0.999"} 15 | {"data": "0.9999"} 16 | {"data": "+1.0"} 17 | {"data": "-1.0"} 18 | {"data": "1 . 0"} 19 | {"data": "- 1 . 0"} 20 | {"data": "+ 1 . 0"} 21 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/floats.json: -------------------------------------------------------------------------------- 1 | { "number": -3.141592 } 2 | { "number": 3.141592 } 3 | { "number": 0.0 } 4 | { "number": -0.0 } 5 | { "number": -3.4028234663852886e+38 } -------------------------------------------------------------------------------- /integration_tests/src/test/resources/floats_edge_cases.json: -------------------------------------------------------------------------------- 1 | { "number": "-3.141592" } 2 | { "number": "3.141592" } 3 | { "number": "-3.4028234663852886e+38" } 4 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/floats_invalid.csv: -------------------------------------------------------------------------------- 1 | "number" 2 | true 3 | false 4 | bad 5 | "bad" -------------------------------------------------------------------------------- /integration_tests/src/test/resources/floats_invalid.json: -------------------------------------------------------------------------------- 1 | { "number": true } 2 | { "number": "not a float" } 3 | { "number": "" } -------------------------------------------------------------------------------- /integration_tests/src/test/resources/floats_leading_zeros.json: -------------------------------------------------------------------------------- 1 | { "number": -03.141592 } 2 | { "number": 3.141592 } 3 | { "number": 10 } 4 | { "number": -00.0 } 5 | { "number": 000.012 } 6 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/hive-delim-text/carriage-return-err/carriage.return.err.hive.txt: -------------------------------------------------------------------------------- 1 | abc def ghi 2 | ABC DEF GHI 3 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/hive-delim-text/carriage-return/carriage.return.hive.txt: -------------------------------------------------------------------------------- 1 | abc def ghi 2 | ABC DEF GHI 3 | JKLMNO PQR 4 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/hive-delim-text/comments/comments.hive.txt: -------------------------------------------------------------------------------- 1 | All3And 2 | # the4the 3 | ## leaves9sky 4 | are3is 5 | # brown7grey 6 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/hive-delim-text/date/date.hive.txt: -------------------------------------------------------------------------------- 1 | 2020-09-16 2 | 2020-10-16 3 | 2021-09-16 4 | 2021-40-32 5 | AnotherObviouslyInvalidDate 6 | 2022/09/16 7 | 2021-09 8 | 2021/09 9 | 09/2021 10 | 09-2021 11 | 09-16-2021 12 | 09/16/2021 13 | 2021-09-16T 14 | 2021-09-16 23:59:59 15 | 2021-09-16T23:59:59 -------------------------------------------------------------------------------- /integration_tests/src/test/resources/hive-delim-text/extended-float-values/ext_float_values.hive.txt: -------------------------------------------------------------------------------- 1 | 1.7976931348623157E-308 2 | 1.7976931348623157e-100 3 | 1.2e-234 4 | 1111111111111111111111111111111111111111E-39 5 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/hive-delim-text/simple-boolean-values/bools.hive.txt: -------------------------------------------------------------------------------- 1 | "booleans" 2 | true 3 | false 4 | \N 5 | True 6 | False 7 | TRUE 8 | FALSE 9 | BAD 10 | y 11 | n 12 | yes 13 | no 14 | 1 15 | 0 16 | t 17 | f 18 | true 19 | true 20 | false 21 | false 22 | 1 23 | 1 24 | 0 25 | 0 26 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/hive-delim-text/simple-float-values/simple_float_values.hive.txt: -------------------------------------------------------------------------------- 1 | number 2 | 1.042 3 | -- 4 | bad 5 | "" 6 | , 7 | 98.343 8 | 223823.9484 9 | 23848545.0374 10 | 184721.23987223 11 | 3.4028235e+38 12 | -0.0 13 | 3.4028235E38 14 | 3.4028236E38 15 | 3.4028236e+38 16 | 3.4028236e-38 17 | 1.7976931348623157E99 18 | 1.7976931348623157e+99 19 | 1.7976931348623157e-99 20 | 1.2 21 | 1.2 22 | 1 23 | 1 24 | 1.7976931348623157E308 25 | 1.7976931348623157e+308 26 | 1.111111111111111111111111111111111111111E-39 27 | 1.11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111 28 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/hive-delim-text/simple-int-values/simple_int_values.hive.txt: -------------------------------------------------------------------------------- 1 | number 2 | -1 3 | 0 4 | 1 5 | 127 6 | 128 7 | -128 8 | -129 9 | 32767 10 | -32768 11 | 32768 12 | -32769 13 | 2147483647 14 | -2147483648 15 | 2147483648 16 | -2147483649 17 | 9223372036854775807 18 | -9223372036854775808 19 | 9223372036854775808 20 | -9223372036854775809 21 | 18446744073709551615 22 | A 23 | B 24 | C 25 | +50 26 | \n 27 | + 28 | - 29 | 100 30 | 100 31 | BAD 32 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/hive-delim-text/timestamp/ts.hive.txt: -------------------------------------------------------------------------------- 1 | 2020-09-16 2 | 2020-09-16T22:32 3 | 2020-09-16 22:32 4 | 2020-09-16T22:32:01 5 | 2020-09-16T22:32:01 6 | 2020-09-16T22:32:01 7 | 2020-09-16 22:32:01 8 | 2020-09-16 22:32:01.1 9 | 2020-09-16 22:32:01.12 10 | 2020-09-16 22:32:01.123 11 | 2020-09-16 22:32:01.1234 12 | 2020-09-16 22:32:01.12345 13 | 2020-09-16 22:32:01.123456 14 | 2020-09-16 22:32:01.1234567 15 | 2020-09-16T22:32:01Z 16 | 2020-09-16 22:32:01Z 17 | 2020-90-16 22:32:01Z 18 | Patently not a timestamp row 19 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/hive-delim-text/trucks-1/trucks.hive.txt: -------------------------------------------------------------------------------- 1 | FordF-150202030135.15Luxury/Fullsize 2 | GMCSierra 1500199720000No comment 3 | ChevyD-Max20155,000.10 -------------------------------------------------------------------------------- /integration_tests/src/test/resources/hive-delim-text/trucks-err/trucks.err.hive.txt: -------------------------------------------------------------------------------- 1 | "Ford"F-150202030135.15Luxury/Fullsize 2 | 3 | GMC"Sierra 1500"199720000No comment 4 | ChevyD-Max20155,000.10 5 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/int_array_formatted.json: -------------------------------------------------------------------------------- 1 | {"data": [1,0]} 2 | {"data": [-1,,100]} 3 | {"data": [-0,5,6,7 ,8 , 9 ]} 4 | {"data": [0]} 5 | {"data": [127, -128]} 6 | {"data": []} 7 | {"data": [32767, -32768]} 8 | {"data": [2147483647, -2147483648]} 9 | {"data": [9223372036854775807,-9223372036854775808]} 10 | {"data": [9223372036854775808, -9223372036854775809]} 11 | {"data": [99999999999999999999999999999999999999, -99999999999999999999999999999999999999]} 12 | {"data": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]} 13 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/int_formatted.json: -------------------------------------------------------------------------------- 1 | {"data": 1} 2 | {"data": -1} 3 | {"data": -0} 4 | {"data": 0} 5 | {"data": 127} 6 | {"data": -128} 7 | {"data": 32767} 8 | {"data": -32768} 9 | {"data": 2147483647} 10 | {"data": -2147483648} 11 | {"data": 9223372036854775807} 12 | {"data": -9223372036854775808} 13 | {"data": 9223372036854775808} 14 | {"data": 99999999999999999999999999999999999999} 15 | {"data": -99999999999999999999999999999999999999} 16 | {"data": 999999999999999999999999999999999999999} 17 | {"data": -999999999999999999999999999999999999999} 18 | {"data": १} 19 | {"data": ๑} 20 | {"data": १०} 21 | {"data": ๑๐} 22 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/int_formatted_strings.json: -------------------------------------------------------------------------------- 1 | {"data": "1"} 2 | {"data": "-1"} 3 | {"data": "-0"} 4 | {"data": "0"} 5 | {"data": "127"} 6 | {"data": "-128"} 7 | {"data": "32767"} 8 | {"data": "-32768"} 9 | {"data": "2147483647"} 10 | {"data": "-2147483648"} 11 | {"data": "9223372036854775807"} 12 | {"data": "-9223372036854775808"} 13 | {"data": "9223372036854775808"} 14 | {"data": "99999999999999999999999999999999999999"} 15 | {"data": "-99999999999999999999999999999999999999"} 16 | {"data": "999999999999999999999999999999999999999"} 17 | {"data": "-999999999999999999999999999999999999999"} 18 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/int_mixed_array_struct_formatted.json: -------------------------------------------------------------------------------- 1 | {"data": {"A": 0, "B": 1}} 2 | {"data": [1,0]} 3 | {"data": {"A": 1}} 4 | {"data": [-1,,100]} 5 | {"data": {"B": 50}} 6 | {"data": [0]} 7 | {"data": null} 8 | {"data": []} 9 | {"data": {"B": -128, "A": 127}} 10 | {"data": [127, -128]} 11 | {"data": {"A": 32767, "B": -32767}} 12 | {"data": [32767, -32768]} 13 | {"data": {"A": 214783647, "B": -2147483648}} 14 | {"data": [2147483647, -2147483648]} 15 | {"data": {"A": 9223372036854775807, "B": -9223372036854775808}} 16 | {"data": [9223372036854775807,-9223372036854775808]} 17 | {"data": {"A": 9223372036854775808,, "B": -9223372036854775809}} 18 | {"data": [9223372036854775808, -9223372036854775809]} 19 | {"data": {"B": 99999999999999999999999999999999999999, "A": -99999999999999999999999999999999999999}} 20 | {"data": [99999999999999999999999999999999999999, -99999999999999999999999999999999999999]} 21 | {"data": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]} 22 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/int_struct_formatted.json: -------------------------------------------------------------------------------- 1 | {"data": {"A": 0, "B": 1}} 2 | {"data": {"A": 1}} 3 | {"data": {"B": 50}} 4 | {"data": {"B": -128, "A": 127}} 5 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/int_struct_formatted_problematic_rows.json: -------------------------------------------------------------------------------- 1 | {"data": {"B": 99999999999999999999, "A": -9999999999999999999}} 2 | {"data": {"A": 0, "B": "0"}} 3 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/ints.json: -------------------------------------------------------------------------------- 1 | { "number": 0 } 2 | { "number": -128 } 3 | { "number": 127 } 4 | { "number": -32768 } 5 | { "number": 32767 } 6 | { "number": -2147483648 } 7 | { "number": 2147483647 } 8 | { "number": -9223372036854775808 } 9 | { "number": 9223372036854775807 } -------------------------------------------------------------------------------- /integration_tests/src/test/resources/ints_invalid.json: -------------------------------------------------------------------------------- 1 | { "number": true } 2 | { "number": 3.141 } 3 | { "number": "0" } 4 | { "number": "-128" } 5 | { "number": "127" } 6 | { "number": "-32768" } 7 | { "number": "32767" } 8 | { "number": "-2147483648" } 9 | { "number": "2147483647" } 10 | { "number": "-9223372036854775808" } 11 | { "number": "9223372036854775807" } -------------------------------------------------------------------------------- /integration_tests/src/test/resources/ints_with_whitespace.csv: -------------------------------------------------------------------------------- 1 | "number" 2 | 1 3 | 2 4 | 3, 5 | 4 6 | 5 , 7 | "6" 8 | "7" 9 | "8" 10 | "9" , 11 | 10 12 | 11 13 | 12 14 | "13" , 15 | "14" , 16 | 15 17 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/just_comments.csv: -------------------------------------------------------------------------------- 1 | #This,should,look,like,nothing 2 | #COMMENT 3 | 4 | # More Comments 5 | 6 | 7 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/malformed1.ndjson: -------------------------------------------------------------------------------- 1 | { "number": 1 } 2 | { , } 3 | { : 2 } 4 | { "number": 2 } 5 | { "number": } 6 | not_json 7 | { 4 } 8 | { "number": 5 } 9 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/malformed2.ndjson: -------------------------------------------------------------------------------- 1 | { "number": 1 } 2 | { "number": 2 } 3 | { "number": 3 4 | { "number": 4 } 5 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/malformed3.ndjson: -------------------------------------------------------------------------------- 1 | { "number": 1 } 2 | { "number": 2 } 3 | { "number": 3 } { "number": 4 } 4 | { "number": 5 } 5 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/malformed4.ndjson: -------------------------------------------------------------------------------- 1 | { "number": 1 } 2 | { "number": 2 } 3 | { "number": 3 } x 4 | { "number": 5 } 5 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/mixed-primitive-arrays.ndjson: -------------------------------------------------------------------------------- 1 | { "var0": [1, 2, 3] } 2 | { "var0": [false, false, true] } 3 | { "var0": ["one", "two", "three"] } 4 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/mixed-primitives.ndjson: -------------------------------------------------------------------------------- 1 | { "var0": "hello", "var1": 3.141, "var2": true } 2 | "var0": "hello", "var1": 3.141, "var2": true } 3 | { "var0": "hello", "var1": true, "var2": "hello" } 4 | { "var0": true, "var1": "hello", "hello" } 5 | { "var0": true, "var1": "hello", "var2": "hello" } { "var0": true, "var1": "hello", "var2": "hello" } 6 | { "var0": true, "var1": "hello", "var2" } 7 | { "var0": 123, "var1": true, "var2" true } 8 | { "var0": 123, "var1": true "var2": true 9 | { "var0": "hello", "var1": "hello", "var2": "hello" } -------------------------------------------------------------------------------- /integration_tests/src/test/resources/mixed-types-in-struct.ndjson: -------------------------------------------------------------------------------- 1 | { "var0": [1, 2, 3] } 2 | { "var0": { "a": 54321} } 3 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/nan_and_inf.csv: -------------------------------------------------------------------------------- 1 | "number" 2 | NaN 3 | Inf 4 | +Inf 5 | -Inf 6 | NAN 7 | nan 8 | INF 9 | +INF 10 | -INF 11 | Infinity 12 | +Infinity 13 | -Infinity 14 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/nan_and_inf.json: -------------------------------------------------------------------------------- 1 | { "number": NaN } 2 | { "number": +INF } 3 | { "number": -INF } 4 | { "number": Infinity } 5 | { "number": +Infinity } 6 | { "number": -Infinity } -------------------------------------------------------------------------------- /integration_tests/src/test/resources/nan_and_inf_invalid.json: -------------------------------------------------------------------------------- 1 | { "number": NAN } 2 | { "number": nan } 3 | { "number": INF } 4 | { "number": Inf } 5 | { "number": +Inf } 6 | { "number": -Inf } 7 | { "number": "NAN" } 8 | { "number": "nan" } 9 | { "number": "INF" } 10 | { "number": "Inf" } 11 | { "number": "+Inf" } 12 | { "number": "-Inf" } -------------------------------------------------------------------------------- /integration_tests/src/test/resources/nan_and_inf_strings.json: -------------------------------------------------------------------------------- 1 | { "number": "NaN" } 2 | { "number": "+INF" } 3 | { "number": "-INF" } 4 | { "number": "Infinity" } 5 | { "number": "+Infinity" } 6 | { "number": "-Infinity" } -------------------------------------------------------------------------------- /integration_tests/src/test/resources/nested-structs.ndjson: -------------------------------------------------------------------------------- 1 | { "teacher": "Bob" } 2 | { "student": { "name": "Carol", "age": 21 } } 3 | { "teacher": "Bob", "student": { "name": "Carol", "age": 21 } } 4 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/no-comments.csv: -------------------------------------------------------------------------------- 1 | ~1,2,3 2 | #4,5,6 3 | 7,8,9 4 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/optional-fields.ndjson: -------------------------------------------------------------------------------- 1 | { "teacher": "Bob" } 2 | { "student": "Carol" } 3 | { "teacher": "Bob", "student": "Carol" } 4 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/parquet_acq/part-00000-acquisition.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/integration_tests/src/test/resources/parquet_acq/part-00000-acquisition.snappy.parquet -------------------------------------------------------------------------------- /integration_tests/src/test/resources/parquet_perf/part-00000-performance.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/integration_tests/src/test/resources/parquet_perf/part-00000-performance.snappy.parquet -------------------------------------------------------------------------------- /integration_tests/src/test/resources/repeated_columns.json: -------------------------------------------------------------------------------- 1 | {"data":1,"data":0} 2 | {"data":-1,"data":50} 3 | {"data":0,"data":"string"} 4 | {"data":127,"data":"127"} 5 | {"data":100,"data":200} 6 | {"data":null,"data":200} 7 | {"data":100,"data":null} 8 | {"data":null,"data":false} 9 | {"data":true,"data":false} 10 | {"data":100.012,"data":null} 11 | {"data":100,"data":{"a":50,"a":100}} 12 | {"data":{"b":100},"data":{"a":200}} 13 | {"data":{"a":100},"data":{"b":200}} 14 | {"data":{"a":101,"a":102},"data":null} 15 | {"data":{"a":100},"data":"test"} 16 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/scan_emtpy_lines.json: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | {"BAD"} 6 | 7 | 8 | 9 | 10 | {"BAD"} 11 | 12 | 13 | 14 | 15 | {"BAD"} 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/sci_formatted.json: -------------------------------------------------------------------------------- 1 | {"data": 1E0} 2 | {"data": 1E1} 3 | {"data": -1E1} 4 | {"data": 1E308} 5 | {"data": 1.7976931348623157E308} 6 | {"data": -1.7976931348623157E308} 7 | {"data": 3.4028235E38} 8 | {"data": -3.4028235E38} 9 | {"data": 1E309} 10 | {"data": 1E-309} 11 | {"data": +1E0} 12 | {"data": +1E1} 13 | {"data": 1E-1} 14 | {"data": 1E+1} 15 | {"data": 1e1} 16 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/sci_formatted_strings.json: -------------------------------------------------------------------------------- 1 | {"data": "1E0"} 2 | {"data": "1E1"} 3 | {"data": "-1E1"} 4 | {"data": "1E308"} 5 | {"data": "1.7976931348623157E308"} 6 | {"data": "-1.7976931348623157E308"} 7 | {"data": "3.4028235E38"} 8 | {"data": "-3.4028235E38"} 9 | {"data": "1E309"} 10 | {"data": "1E-309"} 11 | {"data": "+1E0"} 12 | {"data": "+1E1"} 13 | {"data": "1E-1"} 14 | {"data": "1E+1"} 15 | {"data": "1e1"} 16 | {"data": "1 E 1"} 17 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/simple_boolean_values.csv: -------------------------------------------------------------------------------- 1 | "booleans" 2 | true 3 | false 4 | 5 | True 6 | False 7 | TRUE 8 | FALSE 9 | BAD 10 | y 11 | n 12 | yes 13 | no 14 | 1 15 | 0 16 | t 17 | f 18 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/simple_float_values.csv: -------------------------------------------------------------------------------- 1 | "number" 2 | 1.042 3 | -- 4 | bad 5 | "" 6 | , 7 | 98.343 8 | 223823.9484 9 | 23848545.0374 10 | 184721.23987223 11 | 3.4028235e+38 12 | -0.0 13 | 3.4028235E38 14 | 3.4028236E38 15 | 3.4028236e+38 16 | 1.7976931348623157E308 17 | 1.7976931348623157e+308 18 | 1.2e-234 -------------------------------------------------------------------------------- /integration_tests/src/test/resources/simple_int_values.csv: -------------------------------------------------------------------------------- 1 | "number" 2 | -1 3 | 0 4 | 1 5 | 127 6 | 128 7 | -128 8 | -129 9 | 32767 10 | -32768 11 | 32768 12 | -32769 13 | 2147483647 14 | -2147483648 15 | 2147483648 16 | -2147483649 17 | 9223372036854775807 18 | -9223372036854775808 19 | 9223372036854775808 20 | -9223372036854775809 21 | 18446744073709551615 22 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/single_quoted_strings.json: -------------------------------------------------------------------------------- 1 | {"data":'test'} 2 | {"data":'"test'} 3 | {'data':"test"} 4 | {'data':"'test"} 5 | {"data":"test",'other':'other test'} 6 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/small_float_values.csv: -------------------------------------------------------------------------------- 1 | "number" 2 | 0.001 3 | 0.01 4 | 0.1 5 | 0.2 6 | 0.3 7 | 0.4 8 | 1.0 9 | 2.0 10 | 3.0 11 | 4.0 12 | -0.001 13 | -0.01 14 | -0.1 15 | -0.2 16 | -0.3 17 | -0.4 18 | -1.0 19 | -2.0 20 | -3.0 21 | -4.0 -------------------------------------------------------------------------------- /integration_tests/src/test/resources/str.csv: -------------------------------------------------------------------------------- 1 | Something 2 | 2020-09-16 3 | 2020-09-16T22:32 4 | 2020-09-16T22:32:01 5 | 2020-09-16T22:32:01Z 6 | "FOO\rBAR" 7 | TEST\,MORE 8 | LAST ONE 9 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/test_orc_varchar.orc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/integration_tests/src/test/resources/test_orc_varchar.orc -------------------------------------------------------------------------------- /integration_tests/src/test/resources/timestamp-date-test.orc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/integration_tests/src/test/resources/timestamp-date-test.orc -------------------------------------------------------------------------------- /integration_tests/src/test/resources/timestamp-nanos.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/integration_tests/src/test/resources/timestamp-nanos.parquet -------------------------------------------------------------------------------- /integration_tests/src/test/resources/timestamp_tz_formatted_strings.json: -------------------------------------------------------------------------------- 1 | {"data": "2024-08-22T14:45:00-05:00"} 2 | {"data": "2023-02-28T09:30:00+01:00"} 3 | {"data": "1969-07-20T20:17:40-04:00"} 4 | {"data": "1970-01-01T00:00:00+00:00"} 5 | {"data": "1999-12-31T23:59:59-08:00"} 6 | {"data": "2012-06-30T23:59:60+09:00"} 7 | {"data": "1945-05-08T02:01:00+02:00"} 8 | {"data": "2024-08-22T14:45:00+10:00"} 9 | {"data": "2023-02-28T14:45:00-03:30"} 10 | {"data": "2021-12-31T23:59:59+05:30"} 11 | {"data": "2023-02-28T14:45:00.123-05:00"} 12 | 13 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/timestamps.json: -------------------------------------------------------------------------------- 1 | { "number": "2020-02-25" } 2 | { "number": "2020-02-25 14:46" } 3 | { "number": "2020-02-25T14:46" } 4 | { "number": "2020-02-25 14:46:00" } 5 | { "number": "2020-02-25T14:46:00" } 6 | { "number": "2020-02-25T14:46:00 " } 7 | { "number": "2020-02-25 14:46:00.123" } 8 | { "number": "2020-02-25T14:46:00.123" } 9 | { "number": " 2020-02-25T14:46:00.123" } 10 | { "number": "2020-02-25 14:46:00.123456" } 11 | { "number": "2020-02-25T14:46:00.123456" } 12 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/trucks-blank-names.csv: -------------------------------------------------------------------------------- 1 | "",,year,price,comment,extra 2 | "Ford","F-150","2020","30,135.15","Luxury/Fullsize" 3 | 4 | GMC,Sierra 1500,1997,20000,"No comment", 5 | Chevy,D-Max,2015,"5,000.10" 6 | 7 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/trucks-comments.csv: -------------------------------------------------------------------------------- 1 | ~ Some metadata 2 | ~Comment at the start of the line 3 | make,model,year,price,comment,extra 4 | "Ford","F-150","2020","30,135.15","Luxury/Fullsize" 5 | 6 | ~GMC,Sierra 1500,1997,20000,"No comment", 7 | Chevy,D-Max,2015,"5,000.10" 8 | ~ Last comment line 9 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/trucks-different.csv: -------------------------------------------------------------------------------- 1 | make|model|year|price|comment|extra 2 | 'Ford'|'F-150'|'2020'|'30,135.15'|'Luxury/Fullsize' 3 | 4 | GMC|Sierra 1500|1997|20000|'No comment'| 5 | Chevy|D-Max|2015|5,000.10 6 | 7 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/trucks-empty-values.csv: -------------------------------------------------------------------------------- 1 | 2 | make,model,year,price,comment,extra 3 | "Ford","","2020",,"Luxury/Fullsize" 4 | 5 | GMC,Sierra 1500,1997,20000,"No comment", 6 | Chevy,D-Max,2015,,"" 7 | 8 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/trucks-extra-columns.csv: -------------------------------------------------------------------------------- 1 | 2 | make,model,year,price,comment,extra 3 | "Ford","F-150","2020","30,135.15","Luxury/Fullsize",,,,,, 4 | 5 | GMC,Sierra 1500,1997,20000,"No comment",,, 6 | Chevy,D-Max,2015,"5,000.10",,,,,,, 7 | 8 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/trucks-missing-quotes.csv: -------------------------------------------------------------------------------- 1 | 2 | make,model,year,price,comment,extra 3 | "Ford,F-150,2020,30,135.15,Luxury/Fullsize 4 | 5 | GMC,Sierra 1500,1997,20000,No comment", 6 | "Chevy,"D-Max",2015,5,000.10 7 | 8 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/trucks-more-comments.csv: -------------------------------------------------------------------------------- 1 | # Some metadata 2 | #Comment at the start of the line 3 | make,model,year,price,comment,extra 4 | "Ford","F-150","2020","30,135.15","Luxury/Fullsize" 5 | #GMC,Sierra 1500,1997,20000,"No comment", 6 | 7 | Chevy,D-Max,2015,"5,000.10", 8 | # Last comment line 9 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/trucks-null.csv: -------------------------------------------------------------------------------- 1 | 2 | make,model,year,price,comment,extra 3 | "Ford",null,"2020","30,135.15",null 4 | 5 | GMC,Sierra 1500,1997,20000,"No comment", 6 | Chevy,D-Max,null,"5,000.10" 7 | 8 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/trucks-windows.csv: -------------------------------------------------------------------------------- 1 | 2 | make,model,year,price,comment,extra 3 | "Ford","F-150","2020","30,135.15","Luxury/Fullsize" 4 | 5 | GMC,Sierra 1500,1997,20000,"No comment", 6 | Chevy,D-Max,2015,"5,000.10" 7 | 8 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/trucks.csv: -------------------------------------------------------------------------------- 1 | 2 | make,model,year,price,comment,extra 3 | "Ford","F-150","2020","30,135.15","Luxury/Fullsize" 4 | 5 | GMC,Sierra 1500,1997,20000,"No comment", 6 | Chevy,D-Max,2015,"5,000.10" 7 | 8 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/trucks.tsv: -------------------------------------------------------------------------------- 1 | make model year price comment extra 2 | "Ford" "F-150" "2020" "30,135.15" "Luxury/Fullsize" 3 | GMC Sierra 1500 1997 20000 "No comment" 4 | Chevy D-Max 2015 5,000.10 5 | 6 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/ts.csv: -------------------------------------------------------------------------------- 1 | 2020-09-16 2 | 2020-09-16T22:32 3 | 2020-09-16 22:32 4 | 2020-09-16T22:32:01 5 | 2020-09-16T22:32:01 6 | 2020-09-16T22:32:01 7 | 2020-09-16 22:32:01 8 | 2020-09-16T22:32:01Z 9 | 2020-09-16 22:32:01Z 10 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/unquotedChars.json: -------------------------------------------------------------------------------- 1 | { "a": "quotedChar-27" } 2 | { "a": "quotedChar-7" } 3 | { "a": "quotedChar-9 " } 4 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/withBackslashEscapingAnyCharacter.json: -------------------------------------------------------------------------------- 1 | {"str": "value"} 2 | {"str": "value \u0035"} 3 | {"\str": "value"} 4 | {"st\r": "value"} 5 | {"str\": "value"} 6 | {"str": "\value"} 7 | {"str": "v\alue"} 8 | {"str": "va\lue"} 9 | {"str": "val\ue"} 10 | {"str": "valu\e"} 11 | {"str": "\$value"} 12 | {"str"\: "value"} 13 | {"str": "value"\} 14 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/withComments.json: -------------------------------------------------------------------------------- 1 | {"str": /* Hello */ "World"} 2 | {"str": /*TEST "TEST"*/ "TEST2"} 3 | {"str": "TEST3"} // COMMENT AFTER THE DATA 4 | {"str": "DATA"} 5 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/withNonnumericNumbers.json: -------------------------------------------------------------------------------- 1 | {"double": +Infinity, "float": +Infinity} 2 | {"double": Infinity, "float": Infinity} 3 | {"double": -Infinity, "float": -Infinity} 4 | {"double": +INF, "float": +INF} 5 | {"double": INF, "float": INF} 6 | {"double": -INF, "float": -INF} 7 | {"double": NaN, "float": NaN} 8 | {"double": 1, "float": 1} 9 | {"double": "+Infinity", "float": "+Infinity"} 10 | {"double": "Infinity", "float": "Infinity"} 11 | {"double": "-Infinity", "float": "-Infinity"} 12 | {"double": "+INF", "float": "+INF"} 13 | {"double": "INF", "float": "INF"} 14 | {"double": "-INF", "float": "-INF"} 15 | {"double": "NaN", "float": "NaN"} 16 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/withNumericLeadingZeros.json: -------------------------------------------------------------------------------- 1 | {"byte": 01} 2 | {"byte": 001} 3 | {"byte": -01} 4 | {"byte": 01234} 5 | {"byte": 0123} 6 | {"byte": 1} 7 | {"int": 01} 8 | {"int": 001} 9 | {"int": -01} 10 | {"int": 01234} 11 | {"int": 0123} 12 | {"int": 1} 13 | {"float": 01.0} 14 | {"float": 001.01} 15 | {"float": -01.0} 16 | {"float": 01234} 17 | {"float": 0123.4} 18 | {"float": 01234} 19 | {"float": 0123.4} 20 | {"float": 1} 21 | {"decimal": 01.0} 22 | {"decimal": 001.01} 23 | {"decimal": -01.0} 24 | {"decimal": 01234} 25 | {"decimal": 0123.4} 26 | {"decimal": 1} 27 | {"decimal": "01.0"} 28 | {"decimal": "001.01"} 29 | {"decimal": "-01.0"} 30 | {"decimal": "01234"} 31 | {"decimal": "0123.4"} 32 | {"decimal": "1"} 33 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/withSingleQuotes.json: -------------------------------------------------------------------------------- 1 | {"str": 'World'} 2 | {'str': "TEST2"} 3 | {"str": "TEST3", "other": 'something'} 4 | {"str": "DATA"} 5 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/withUnquotedControlChars.json: -------------------------------------------------------------------------------- 1 | {"str": "value"} 2 | {"str": "value"} 3 | {"str": "value"} 4 | {"str": "value"} 5 | {"str": "value"} 6 | {"str": "value", "other": "info"} 7 | -------------------------------------------------------------------------------- /integration_tests/src/test/resources/withUnquotedFieldNames.json: -------------------------------------------------------------------------------- 1 | {str: "World"} 2 | {str: "TEST2"} 3 | {"str": "TEST3", other: "MORE"} 4 | {"str": "DATA"} 5 | -------------------------------------------------------------------------------- /integration_tests/src/test/scala/com/nvidia/spark/rapids/tests/mortgage/MortgageAdaptiveSparkSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.nvidia.spark.rapids.tests.mortgage 17 | 18 | class MortgageAdaptiveSparkSuite extends MortgageSparkSuite { 19 | override def adaptiveQueryEnabled: Boolean = true 20 | } 21 | -------------------------------------------------------------------------------- /integration_tests/src/test/spark400/scala/org/apache/spark/sql/rapids/shims/TrampolineUtilShim.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "400"} 19 | spark-rapids-shim-json-lines ***/ 20 | 21 | package org.apache.spark.sql.rapids.shims 22 | 23 | import org.apache.spark.sql.classic.SparkSession 24 | 25 | object TrampolineUtilShim { 26 | 27 | def cleanupAnyExistingSession() = SparkSession.cleanupAnyExistingSession() 28 | } 29 | -------------------------------------------------------------------------------- /jenkins/databricks/init_cuda11_runtime.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright (c) 2021-2023, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # The init script to install cuda11.0 toolkit 19 | # Will be automatically pushed into the Databricks workspace: /databricks/init_scripts once it is updated. 20 | 21 | wget http://developer.download.nvidia.com/compute/cuda/11.0.2/local_installers/cuda_11.0.2_450.51.05_linux.run 22 | 23 | sh cuda_11.0.2_450.51.05_linux.run --toolkit --silent 24 | 25 | rm cuda_11.0.2_450.51.05_linux.run 26 | -------------------------------------------------------------------------------- /jenkins/test-timezones.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright (c) 2023-2025, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | set -ex 19 | 20 | # Test cases are used in both pre-merge test and nightly-build test. 21 | # We choose up to 7 cases so as to run through all time zones every week during nightly-build test. 22 | export time_zones_test_cases=( 23 | "Asia/Shanghai" # CST 24 | "America/New_York" # PST 25 | ) 26 | -------------------------------------------------------------------------------- /python/rapids/__init__.py: -------------------------------------------------------------------------------- 1 | ## 2 | # Copyright (c) 2020, NVIDIA CORPORATION. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | ## 17 | -------------------------------------------------------------------------------- /sql-plugin-api/src/main/scala/com/nvidia/spark/rapids/DFUDFPluginAPI.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.nvidia.spark.rapids 18 | 19 | import org.apache.spark.sql.{SparkSession, SparkSessionExtensions} 20 | import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan 21 | import org.apache.spark.sql.catalyst.rules.Rule 22 | 23 | trait DFUDFPluginAPI { 24 | def apply(extensions: SparkSessionExtensions): Unit 25 | 26 | def logicalPlanRules(sparkSession: SparkSession): Rule[LogicalPlan] 27 | } 28 | -------------------------------------------------------------------------------- /sql-plugin-api/src/main/scala/com/nvidia/spark/rapids/SparkShimServiceProvider.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020-2022, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.nvidia.spark.rapids 18 | 19 | /** 20 | * A Spark version shim layer interface. 21 | */ 22 | trait SparkShimServiceProvider { 23 | def getShimVersion: ShimVersion 24 | def matchesVersion(version:String): Boolean 25 | } 26 | -------------------------------------------------------------------------------- /sql-plugin-api/src/main/scala/org/apache/spark/sql/rapids/execution/UnshimmedTrampolineUtil.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.apache.spark.sql.rapids.execution 18 | 19 | object UnshimmedTrampolineUtil { 20 | def sparkClassLoader: ClassLoader = { 21 | org.apache.spark.util.Utils.getContextOrSparkClassLoader 22 | } 23 | 24 | def classIsLoadable(className: String): Boolean = 25 | org.apache.spark.util.Utils.classIsLoadable(className) 26 | } 27 | -------------------------------------------------------------------------------- /sql-plugin/src/main/format/README.md: -------------------------------------------------------------------------------- 1 | # FlatBuffers Classes 2 | 3 | This directory contains the FlatBuffers schema files for messages used by 4 | the RAPIDS Spark plugin. 5 | 6 | ## Building the FlatBuffers Compiler 7 | 8 | Build the compiler 1.11.0 version: 9 | ``` 10 | git clone https://github.com/google/flatbuffers.git 11 | cd flatbuffers 12 | git checkout 1.11.0 13 | mkdir build 14 | cd build 15 | cmake .. 16 | make 17 | ``` 18 | 19 | You can install it or otherwise use `flatbuffers/build/flatc` 20 | 21 | ## Generating the FlatBuffers Classes 22 | 23 | From this directory: 24 | 25 | ``` 26 | flatc --java -o ../java/ --gen-mutable *.fbs 27 | ``` 28 | -------------------------------------------------------------------------------- /sql-plugin/src/main/format/ShuffleMetadataRequest.fbs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019-2021, NVIDIA CORPORATION. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | namespace com.nvidia.spark.rapids.format; 16 | 17 | /// Metadata about block IDs to fetch 18 | table BlockIdMeta { 19 | shuffle_id: int; 20 | map_id: long; 21 | start_reduce_id: int; 22 | end_reduce_id: int; 23 | } 24 | 25 | /// Flat buffer for Rapids UCX Shuffle Metadata Request. 26 | table MetadataRequest { 27 | /// array of shuffle block descriptors for which metadata is needed 28 | block_ids : [BlockIdMeta]; 29 | } 30 | 31 | 32 | root_type MetadataRequest; 33 | -------------------------------------------------------------------------------- /sql-plugin/src/main/format/ShuffleTransferRequest.fbs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020-2021, NVIDIA CORPORATION. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | namespace com.nvidia.spark.rapids.format; 16 | 17 | table BufferTransferRequest { 18 | /// ID of table to transfer 19 | buffer_id: int; 20 | } 21 | 22 | /// Flat buffer for Rapids UCX Shuffle Transfer Request. 23 | table TransferRequest { 24 | /// ID used to disambiguate Active Message receives 25 | id : long; 26 | 27 | /// array of table requests to transfer 28 | requests : [BufferTransferRequest]; 29 | } 30 | 31 | root_type TransferRequest; 32 | -------------------------------------------------------------------------------- /sql-plugin/src/main/java/com/nvidia/spark/rapids/WithTableBuffer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.nvidia.spark.rapids; 18 | 19 | import ai.rapids.cudf.DeviceMemoryBuffer; 20 | 21 | /** An interface for obtaining the device buffer backing a contiguous/packed table */ 22 | public interface WithTableBuffer { 23 | DeviceMemoryBuffer getTableBuffer(); 24 | } 25 | -------------------------------------------------------------------------------- /sql-plugin/src/main/java/com/nvidia/spark/rapids/format/CodecType.java: -------------------------------------------------------------------------------- 1 | // automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | package com.nvidia.spark.rapids.format; 4 | 5 | public final class CodecType { 6 | private CodecType() { } 7 | /** 8 | * data simply copied, codec is only for testing 9 | */ 10 | public static final byte COPY = -1; 11 | /** 12 | * no compression codec was used on the data 13 | */ 14 | public static final byte UNCOMPRESSED = 0; 15 | /** 16 | * data compressed with the nvcomp LZ4 codec 17 | */ 18 | public static final byte NVCOMP_LZ4 = 1; 19 | /** 20 | * data compressed with the nvcomp ZSTD codec 21 | */ 22 | public static final byte NVCOMP_ZSTD = 2; 23 | 24 | public static final String[] names = { "COPY", "UNCOMPRESSED", "NVCOMP_LZ4", "NVCOMP_ZSTD", }; 25 | 26 | public static String name(int e) { return names[e - COPY]; } 27 | } 28 | 29 | -------------------------------------------------------------------------------- /sql-plugin/src/main/java/com/nvidia/spark/rapids/format/TransferState.java: -------------------------------------------------------------------------------- 1 | // automatically generated by the FlatBuffers compiler, do not modify 2 | 3 | package com.nvidia.spark.rapids.format; 4 | 5 | public final class TransferState { 6 | private TransferState() { } 7 | /** 8 | * UCX transfer initiated on sender-side 9 | */ 10 | public static final byte STARTED = 0; 11 | /** 12 | * Data has been compressed and requires meta update 13 | */ 14 | public static final byte BUFFER_META_UPDATED = 1; 15 | 16 | public static final String[] names = { "STARTED", "BUFFER_META_UPDATED", }; 17 | 18 | public static String name(int e) { return names[e]; } 19 | } 20 | 21 | -------------------------------------------------------------------------------- /sql-plugin/src/main/scala/com/nvidia/spark/TimingUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.nvidia.spark 18 | 19 | import java.util.concurrent.TimeUnit.NANOSECONDS 20 | 21 | object TimingUtils { 22 | /** Records the duration of running `body`. */ 23 | def timeTakenMs[T](body: => T): (T, Long) = { 24 | val startTime = System.nanoTime() 25 | val result = body 26 | val endTime = System.nanoTime() 27 | (result, math.max(NANOSECONDS.toMillis(endTime - startTime), 0)) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /sql-plugin/src/main/scala/com/nvidia/spark/rapids/AdaptiveSparkPlanHelperImpl.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.nvidia.spark.rapids 18 | 19 | import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper 20 | import org.apache.spark.sql.rapids.AdaptiveSparkPlanHelperShim 21 | 22 | class AdaptiveSparkPlanHelperImpl extends AdaptiveSparkPlanHelperShim with AdaptiveSparkPlanHelper -------------------------------------------------------------------------------- /sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuScan.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.nvidia.spark.rapids 18 | 19 | import org.apache.spark.sql.connector.read.Scan 20 | 21 | trait GpuScan extends Scan with ScanWithMetrics { 22 | /** Create a version of this scan with input file name support */ 23 | def withInputFile(): GpuScan 24 | } 25 | 26 | // Allows use of GpuScan from Java code 27 | abstract class GpuScanWrapper extends GpuScan 28 | -------------------------------------------------------------------------------- /sql-plugin/src/main/scala/com/nvidia/spark/rapids/sparkRapidsListeners.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.nvidia.spark.rapids 18 | 19 | import org.apache.spark.scheduler.SparkListenerEvent 20 | 21 | case class SparkRapidsBuildInfoEvent( 22 | sparkRapidsBuildInfo: Map[String, String], 23 | sparkRapidsJniBuildInfo: Map[String, String], 24 | cudfBuildInfo: Map[String, String], 25 | sparkRapidsPrivateBuildInfo: Map[String, String] 26 | ) extends SparkListenerEvent 27 | -------------------------------------------------------------------------------- /sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/FilteredPartitions.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-2024, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "320"} 19 | {"spark": "321"} 20 | {"spark": "321cdh"} 21 | {"spark": "322"} 22 | {"spark": "323"} 23 | {"spark": "324"} 24 | spark-rapids-shim-json-lines ***/ 25 | package com.nvidia.spark.rapids.shims 26 | 27 | import org.apache.spark.sql.connector.read._ 28 | 29 | trait FilteredPartitions { 30 | @transient protected val filteredPartitions: Seq[InputPartition] 31 | } 32 | -------------------------------------------------------------------------------- /sql-plugin/src/main/spark321/scala/com/nvidia/spark/rapids/shims/SparkShims.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021-2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "321"} 19 | {"spark": "321cdh"} 20 | spark-rapids-shim-json-lines ***/ 21 | package com.nvidia.spark.rapids.shims 22 | 23 | object SparkShimImpl extends Spark321PlusShims 24 | with Spark320PlusNonDBShims 25 | with Spark31Xuntil33XShims 26 | with AnsiCastRuleShims { 27 | override def reproduceEmptyStringBug: Boolean = true 28 | } 29 | -------------------------------------------------------------------------------- /sql-plugin/src/main/spark321cdh/scala/com/nvidia/spark/rapids/shims/GpuParquetCrypto.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022-2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /*** spark-rapids-shim-json-lines 17 | {"spark": "321cdh"} 18 | spark-rapids-shim-json-lines ***/ 19 | package com.nvidia.spark.rapids.shims 20 | 21 | object GpuParquetCrypto { 22 | /** 23 | * Columnar encryption was added in Spark 3.2.0 but CDH doesn't have Parquet 1.12. 24 | */ 25 | def isColumnarCryptoException(e: Throwable): Boolean = false 26 | } 27 | -------------------------------------------------------------------------------- /sql-plugin/src/main/spark322/scala/com/nvidia/spark/rapids/shims/SparkShims.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021-2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "322"} 19 | {"spark": "323"} 20 | {"spark": "324"} 21 | spark-rapids-shim-json-lines ***/ 22 | package com.nvidia.spark.rapids.shims 23 | 24 | object SparkShimImpl extends Spark321PlusShims 25 | with Spark320PlusNonDBShims 26 | with Spark31Xuntil33XShims 27 | with AnsiCastRuleShims 28 | -------------------------------------------------------------------------------- /sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022-2025, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "330"} 19 | {"spark": "330cdh"} 20 | {"spark": "331"} 21 | {"spark": "332"} 22 | {"spark": "332cdh"} 23 | {"spark": "333"} 24 | spark-rapids-shim-json-lines ***/ 25 | package org.apache.spark.sql.rapids.shims 26 | 27 | object RapidsErrorUtils extends RapidsErrorUtils330To334Base 28 | with SequenceSizeTooLongErrorBuilder with ArrayInvalidArgumentErrorUtils 29 | 30 | -------------------------------------------------------------------------------- /sql-plugin/src/main/spark330cdh/scala/com/nvidia/spark/rapids/shims/GpuParquetCrypto.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022-2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /*** spark-rapids-shim-json-lines 17 | {"spark": "330cdh"} 18 | {"spark": "332cdh"} 19 | spark-rapids-shim-json-lines ***/ 20 | package com.nvidia.spark.rapids.shims 21 | 22 | object GpuParquetCrypto { 23 | /** 24 | * Columnar encryption was added in Spark 3.2.0 but CDH doesn't have Parquet 1.12. 25 | */ 26 | def isColumnarCryptoException(e: Throwable): Boolean = false 27 | } 28 | -------------------------------------------------------------------------------- /sql-plugin/src/main/spark330cdh/scala/com/nvidia/spark/rapids/shims/SparkShims.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022-2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "330cdh"} 19 | spark-rapids-shim-json-lines ***/ 20 | package com.nvidia.spark.rapids.shims 21 | 22 | object SparkShimImpl extends Spark33cdhShims {} 23 | -------------------------------------------------------------------------------- /sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/AggregationTagging.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022-2024, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "330db"} 19 | {"spark": "332db"} 20 | {"spark": "341db"} 21 | {"spark": "350db143"} 22 | spark-rapids-shim-json-lines ***/ 23 | package com.nvidia.spark.rapids.shims 24 | 25 | object AggregationTagging { 26 | // Whether aggregations must be replaced only when both halves are replaced. 27 | // Setting this to true due to https://github.com/NVIDIA/spark-rapids/issues/4963 28 | val mustReplaceBoth: Boolean = true 29 | } 30 | -------------------------------------------------------------------------------- /sql-plugin/src/main/spark332cdh/scala/com/nvidia/spark/rapids/shims/SparkShims.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "332cdh"} 19 | spark-rapids-shim-json-lines ***/ 20 | package com.nvidia.spark.rapids.shims 21 | 22 | object SparkShimImpl extends Spark33cdhShims with Spark331PlusNonDBShims {} 23 | -------------------------------------------------------------------------------- /sql-plugin/src/main/spark334/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024-2025, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "334"} 19 | spark-rapids-shim-json-lines ***/ 20 | package org.apache.spark.sql.rapids.shims 21 | 22 | object RapidsErrorUtils extends RapidsErrorUtils330To334Base 23 | with SequenceSizeTooLongUnsuccessfulErrorBuilder with ArrayInvalidArgumentErrorUtils 24 | 25 | -------------------------------------------------------------------------------- /sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-2024, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "340"} 19 | {"spark": "341"} 20 | {"spark": "342"} 21 | {"spark": "343"} 22 | {"spark": "344"} 23 | spark-rapids-shim-json-lines ***/ 24 | package com.nvidia.spark.rapids.shims 25 | 26 | object PartitionedFileUtilsShim extends PartitionedFileUtilsShimBase 27 | -------------------------------------------------------------------------------- /sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/SparkShims.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022-2024, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "340"} 19 | {"spark": "341"} 20 | {"spark": "342"} 21 | {"spark": "343"} 22 | {"spark": "344"} 23 | spark-rapids-shim-json-lines ***/ 24 | package com.nvidia.spark.rapids.shims 25 | 26 | object SparkShimImpl extends Spark340PlusNonDBShims 27 | -------------------------------------------------------------------------------- /sql-plugin/src/main/spark340/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-2025, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "340"} 19 | {"spark": "341"} 20 | {"spark": "350"} 21 | spark-rapids-shim-json-lines ***/ 22 | package org.apache.spark.sql.rapids.shims 23 | 24 | object RapidsErrorUtils extends RapidsErrorUtils340PlusBase 25 | with SequenceSizeTooLongErrorBuilder with ArrayInvalidArgumentErrorUtils 26 | -------------------------------------------------------------------------------- /sql-plugin/src/main/spark341db/scala/com/nvidia/spark/rapids/shims/SparkShims.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-2024, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "341db"} 19 | {"spark": "350db143"} 20 | spark-rapids-shim-json-lines ***/ 21 | package com.nvidia.spark.rapids.shims 22 | 23 | object SparkShimImpl extends Spark341PlusDBShims 24 | -------------------------------------------------------------------------------- /sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022-2025, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "341db"} 19 | spark-rapids-shim-json-lines ***/ 20 | package org.apache.spark.sql.rapids.shims 21 | 22 | object RapidsErrorUtils extends RapidsErrorUtils341DBPlusBase 23 | with SequenceSizeTooLongErrorBuilder with ArrayInvalidArgumentErrorUtils 24 | -------------------------------------------------------------------------------- /sql-plugin/src/main/spark342/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024-2025, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "342"} 19 | {"spark": "343"} 20 | {"spark": "344"} 21 | {"spark": "351"} 22 | {"spark": "352"} 23 | {"spark": "353"} 24 | {"spark": "354"} 25 | {"spark": "355"} 26 | {"spark": "356"} 27 | spark-rapids-shim-json-lines ***/ 28 | package org.apache.spark.sql.rapids.shims 29 | 30 | object RapidsErrorUtils extends RapidsErrorUtils340PlusBase 31 | with SequenceSizeTooLongUnsuccessfulErrorBuilder with ArrayInvalidArgumentErrorUtils 32 | -------------------------------------------------------------------------------- /sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/NullOutputStreamShim.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-2025, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "350"} 19 | {"spark": "350db143"} 20 | {"spark": "351"} 21 | {"spark": "352"} 22 | {"spark": "353"} 23 | {"spark": "354"} 24 | {"spark": "355"} 25 | {"spark": "356"} 26 | {"spark": "400"} 27 | spark-rapids-shim-json-lines ***/ 28 | package com.nvidia.spark.rapids.shims 29 | 30 | import org.apache.commons.io.output.NullOutputStream 31 | 32 | object NullOutputStreamShim { 33 | def INSTANCE = NullOutputStream.INSTANCE 34 | } 35 | -------------------------------------------------------------------------------- /sql-plugin/src/main/spark350db143/scala/com/nvidia/spark/rapids/shims/CastTimeToIntShim.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /*** spark-rapids-shim-json-lines 17 | {"spark": "350db143"} 18 | {"spark": "400"} 19 | spark-rapids-shim-json-lines ***/ 20 | package com.nvidia.spark.rapids.shims 21 | 22 | object CastTimeToIntShim { 23 | // From 400, rows overflow will be set to nulls when casting timestamps to integrals, 24 | // when ANSI is disabled. 25 | def ifNullifyOverflows: Boolean = true 26 | } 27 | -------------------------------------------------------------------------------- /sql-plugin/src/main/spark350db143/scala/com/nvidia/spark/rapids/shims/ShuffleManagerShimUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "350db143"} 19 | {"spark": "400"} 20 | spark-rapids-shim-json-lines ***/ 21 | package com.nvidia.spark.rapids.shims 22 | 23 | object ShuffleManagerShimUtils { 24 | def eagerlyInitialized = true 25 | } 26 | -------------------------------------------------------------------------------- /sql-plugin/src/main/spark350db143/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024-2025, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "350db143"} 19 | spark-rapids-shim-json-lines ***/ 20 | package org.apache.spark.sql.rapids.shims 21 | 22 | object RapidsErrorUtils extends RapidsErrorUtils341DBPlusBase 23 | with SequenceSizeExceededLimitErrorBuilder with ArrayInvalidArgumentErrorUtils 24 | -------------------------------------------------------------------------------- /sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/DateTimeUtilsShims.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "400"} 19 | spark-rapids-shim-json-lines ***/ 20 | 21 | package com.nvidia.spark.rapids.shims 22 | 23 | import org.apache.spark.sql.catalyst.util.SparkDateTimeUtils 24 | 25 | object DateTimeUtilsShims { 26 | def currentTimestamp: Long = SparkDateTimeUtils.instantToMicros(java.time.Instant.now()) 27 | } -------------------------------------------------------------------------------- /sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/NullIntolerantShim.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "400"} 19 | spark-rapids-shim-json-lines ***/ 20 | package com.nvidia.spark.rapids.shims 21 | 22 | import org.apache.spark.sql.catalyst.expressions.Expression 23 | 24 | trait NullIntolerantShim extends Expression { 25 | override def nullIntolerant: Boolean = true 26 | } 27 | -------------------------------------------------------------------------------- /sql-plugin/src/main/spark400/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024-2025, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "400"} 19 | spark-rapids-shim-json-lines ***/ 20 | package org.apache.spark.sql.rapids.shims 21 | 22 | object RapidsErrorUtils extends RapidsErrorUtils340PlusBase 23 | with SequenceSizeExceededLimitErrorBuilder with ArrayInvalidArgumentErrorUtils 24 | -------------------------------------------------------------------------------- /sql-plugin/src/test/scala/com/nvidia/spark/rapids/FQSuiteName.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.nvidia.spark.rapids 18 | 19 | trait FQSuiteName extends org.scalatest.Suite { 20 | override def suiteName: String = getClass().getName() 21 | } 22 | -------------------------------------------------------------------------------- /tests/src/test/resources/dates.csv: -------------------------------------------------------------------------------- 1 | 2019-01-03,1 2 | 2019-01-03,1 3 | 2019-01-03,1 4 | 2019-01-05,2 5 | 2019-01-05,3 6 | 2019-01-06,6 7 | -------------------------------------------------------------------------------- /tests/src/test/resources/day-time-interval-to-be-fix.csv: -------------------------------------------------------------------------------- 1 | interval '106751992' day,,,,interval '2562047789' hour,,,interval '153722867281' minute,, 2 | interval -'106751992' day,,,,interval -'2562047789' hour,,,interval -'153722867281' minute,, 3 | -------------------------------------------------------------------------------- /tests/src/test/resources/decimal-test-legacy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/tests/src/test/resources/decimal-test-legacy.parquet -------------------------------------------------------------------------------- /tests/src/test/resources/decimal-test.orc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/tests/src/test/resources/decimal-test.orc -------------------------------------------------------------------------------- /tests/src/test/resources/decimal-test.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/tests/src/test/resources/decimal-test.parquet -------------------------------------------------------------------------------- /tests/src/test/resources/disorder-read-schema.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/tests/src/test/resources/disorder-read-schema.parquet -------------------------------------------------------------------------------- /tests/src/test/resources/file-splits.orc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/tests/src/test/resources/file-splits.orc -------------------------------------------------------------------------------- /tests/src/test/resources/file-splits.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/tests/src/test/resources/file-splits.parquet -------------------------------------------------------------------------------- /tests/src/test/resources/floats.csv: -------------------------------------------------------------------------------- 1 | 100.0,1.0 2 | 200.0,2.0 3 | 300.0,3.0 4 | 400.0,4.0 5 | 500.0,5.0 6 | -100.0,6.0 7 | -500.0,0.0 8 | -600.0,0.0 9 | -700.0,0.0 10 | -800.0,0.0 11 | -1000.0,0.0 12 | -1500.0,0.0 13 | -2000.0,0.0 14 | -3000.0,0.0 15 | -------------------------------------------------------------------------------- /tests/src/test/resources/hybrid-Julian-calendar.orc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/tests/src/test/resources/hybrid-Julian-calendar.orc -------------------------------------------------------------------------------- /tests/src/test/resources/ints.csv: -------------------------------------------------------------------------------- 1 | -500,0,5,6 2 | -600,0,5,6 3 | -700,0,5,6 4 | -800,0,5,6 5 | -1000,0,5,6 6 | 2147483647,7,5,6 7 | -2147483648,7,5,6 8 | -------------------------------------------------------------------------------- /tests/src/test/resources/lots_o_longs.csv: -------------------------------------------------------------------------------- 1 | 1,1 2 | 2,2 3 | 3,3 4 | 4,4 5 | 5,5 6 | 6,6 7 | 7,7 8 | 8,8 9 | 9,9 10 | 10,10 11 | 11,11 12 | 12,12 13 | 13,13 14 | 14,14 15 | 15,15 16 | 16,16 17 | 17,17 18 | 18,18 19 | 19,19 20 | 20,20 21 | 21,21 22 | 22,22 23 | 23,23 24 | 24,24 25 | 25,25 26 | 26,26 27 | 27,27 28 | 28,28 29 | 29,29 30 | 30,30 31 | 31,31 32 | 32,32 33 | 33,33 34 | 34,34 35 | 35,35 36 | 36,36 37 | 37,37 38 | 38,38 39 | 39,39 40 | 40,40 41 | 41,41 42 | 42,42 43 | 43,43 44 | 44,44 45 | 45,45 46 | 46,46 47 | 47,47 48 | 48,48 49 | 49,49 50 | 50,50 51 | 1,1 52 | 2,2 53 | 3,3 54 | 4,4 55 | 5,5 56 | 6,6 57 | 7,7 58 | 8,8 59 | 9,9 60 | 10,10 61 | 11,11 62 | 12,12 63 | 13,13 64 | 14,14 65 | 15,15 66 | 16,16 67 | 17,17 68 | 18,18 69 | 19,19 70 | 20,20 71 | 21,21 72 | 22,22 73 | 23,23 74 | 24,24 75 | 25,25 76 | 26,26 77 | 27,27 78 | 28,28 79 | 29,29 80 | 30,30 81 | 31,31 82 | 32,32 83 | 33,33 84 | 34,34 85 | 35,35 86 | 36,36 87 | 37,37 88 | 38,38 89 | 39,39 90 | 40,40 91 | 41,41 92 | 42,42 93 | 43,43 94 | 44,44 95 | 45,45 96 | 46,46 97 | 47,47 98 | 48,48 99 | 49,49 100 | 50,50 101 | -------------------------------------------------------------------------------- /tests/src/test/resources/map_of_strings.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/tests/src/test/resources/map_of_strings.snappy.parquet -------------------------------------------------------------------------------- /tests/src/test/resources/mixed-types.csv: -------------------------------------------------------------------------------- 1 | c_string,c_int,c_timestamp 2 | a,1,2020-03-04 00:11:22 3 | a,2,2020-03-04 01:11:22 4 | b,1,2020-03-04 02:11:22 5 | -------------------------------------------------------------------------------- /tests/src/test/resources/nested-unsigned-malformed.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/tests/src/test/resources/nested-unsigned-malformed.parquet -------------------------------------------------------------------------------- /tests/src/test/resources/nested-unsigned.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/tests/src/test/resources/nested-unsigned.parquet -------------------------------------------------------------------------------- /tests/src/test/resources/nullable_floats.csv: -------------------------------------------------------------------------------- 1 | 100.0,1.0, 2 | 200.0,, 3 | 300.0,3.0, 4 | 1.0,4.0, 5 | ,4.0, 6 | 500.0,, 7 | ,6.0, 8 | -500.0,50.5 9 | -------------------------------------------------------------------------------- /tests/src/test/resources/partitioned-csv/partKey=100/part-00000-e2c10641-1db9-47a3-be72-a1a5e94f63c6.c000.csv: -------------------------------------------------------------------------------- 1 | 11,12,13,14,15 2 | 16,17,18,19,20 3 | -------------------------------------------------------------------------------- /tests/src/test/resources/partitioned-csv/partKey=102/part-00000-e2c10641-1db9-47a3-be72-a1a5e94f63c6.c000.csv: -------------------------------------------------------------------------------- 1 | 1,2,3,4,5 2 | 6,7,8,9,10 3 | -------------------------------------------------------------------------------- /tests/src/test/resources/partitioned-parquet/partKey=100/part-00000-7404bd47-cd68-43cf-b78e-5f1c5fba912d.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/tests/src/test/resources/partitioned-parquet/partKey=100/part-00000-7404bd47-cd68-43cf-b78e-5f1c5fba912d.c000.snappy.parquet -------------------------------------------------------------------------------- /tests/src/test/resources/partitioned-parquet/partKey=102/part-00000-7404bd47-cd68-43cf-b78e-5f1c5fba912d.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/tests/src/test/resources/partitioned-parquet/partKey=102/part-00000-7404bd47-cd68-43cf-b78e-5f1c5fba912d.c000.snappy.parquet -------------------------------------------------------------------------------- /tests/src/test/resources/schema-can-prune.orc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/tests/src/test/resources/schema-can-prune.orc -------------------------------------------------------------------------------- /tests/src/test/resources/schema-cant-prune.orc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/tests/src/test/resources/schema-cant-prune.orc -------------------------------------------------------------------------------- /tests/src/test/resources/shorts.csv: -------------------------------------------------------------------------------- 1 | -32768,0,5,6 2 | -1,0,5,6 3 | 32767,1,5,6 4 | 1,1,5,6 5 | 32767,2,5,6 6 | 32767,2,5,6 7 | 16383,3,5,6 8 | 32767,3,5,6 9 | -------------------------------------------------------------------------------- /tests/src/test/resources/strings.csv: -------------------------------------------------------------------------------- 1 | "First","1" 2 | "Foo","2" 3 | "Bar ","3" 4 | ,"4" 5 | "Baz","5" 6 | " Bat ","6" 7 | " Bar","7" 8 | "Bay ","8" -------------------------------------------------------------------------------- /tests/src/test/resources/test.csv: -------------------------------------------------------------------------------- 1 | 1,2,3,4,5 2 | 6,7,8,9,10 3 | 11,12,13,14,15 4 | 16,17,18,19,20 -------------------------------------------------------------------------------- /tests/src/test/resources/test_unsigned64.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/tests/src/test/resources/test_unsigned64.parquet -------------------------------------------------------------------------------- /tests/src/test/resources/test_withheaders.csv: -------------------------------------------------------------------------------- 1 | ints_1,ints_2,ints_3 2 | 1,2,3 3 | 4 | 4,5,6 5 | 6 | 7,8,9 7 | 8 | -------------------------------------------------------------------------------- /tests/src/test/resources/timestamp-date-test-msec.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/tests/src/test/resources/timestamp-date-test-msec.parquet -------------------------------------------------------------------------------- /tests/src/test/resources/timestamp-date-test.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/tests/src/test/resources/timestamp-date-test.parquet -------------------------------------------------------------------------------- /tests/src/test/resources/timestamps.csv: -------------------------------------------------------------------------------- 1 | 2019-01-03T12:34:56.123456,1 2 | 2019-01-03T12:34:56.123456,1 3 | 2019-01-03T12:34:56.123456,1 4 | 2019-01-05T12:34:56.123456,2 5 | 2019-01-05T12:34:56.123456,3 6 | 2019-01-06T12:34:56.123456,6 -------------------------------------------------------------------------------- /tests/src/test/resources/very_large_longs.csv: -------------------------------------------------------------------------------- 1 | 7359046857265451902 2 | 7359046857265451902 3 | 7359046857265451902 4 | -------------------------------------------------------------------------------- /tests/src/test/resources/window-function-test-nonull.orc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/tests/src/test/resources/window-function-test-nonull.orc -------------------------------------------------------------------------------- /tests/src/test/resources/window-function-test.orc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/tests/src/test/resources/window-function-test.orc -------------------------------------------------------------------------------- /tests/src/test/scala/org/apache/spark/sql/FileUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.apache.spark.sql 18 | 19 | import java.io.File 20 | 21 | import org.apache.spark.util.Utils 22 | 23 | object FileUtils { 24 | 25 | /** Generate temp file path and delete it after using */ 26 | def withTempPath(f: File => Unit): Unit = { 27 | val path = Utils.createTempDir() 28 | path.delete() 29 | try f(path) finally Utils.deleteRecursively(path) 30 | } 31 | 32 | def deleteRecursively(path: File): Unit = { 33 | Utils.deleteRecursively(path) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /tests/src/test/scala/org/apache/spark/sql/rapids/TestTrampolineUtil.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-2025, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.apache.spark.sql.rapids 18 | 19 | import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan 20 | import org.apache.spark.sql.rapids.shims.TrampolineConnectShims._ 21 | 22 | object TestTrampolineUtil { 23 | def toLogicalPlan(df: DataFrame): LogicalPlan = df.logicalPlan 24 | 25 | def toDataFrame(spark: SparkSession, plan: LogicalPlan): DataFrame = { 26 | createDataFrame(spark, plan) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsMathFunctionsSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "330"} 19 | spark-rapids-shim-json-lines ***/ 20 | package org.apache.spark.sql.rapids.suites 21 | 22 | import org.apache.spark.sql.MathFunctionsSuite 23 | import org.apache.spark.sql.rapids.utils.RapidsSQLTestsTrait 24 | 25 | class RapidsMathFunctionsSuite extends MathFunctionsSuite with RapidsSQLTestsTrait { 26 | } 27 | -------------------------------------------------------------------------------- /tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsParquetAvroCompatibilitySuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "330"} 19 | spark-rapids-shim-json-lines ***/ 20 | package org.apache.spark.sql.rapids.suites 21 | 22 | import org.apache.spark.sql.execution.datasources.parquet.ParquetAvroCompatibilitySuite 23 | import org.apache.spark.sql.rapids.utils.RapidsSQLTestsBaseTrait 24 | 25 | class RapidsParquetAvroCompatibilitySuite 26 | extends ParquetAvroCompatibilitySuite 27 | with RapidsSQLTestsBaseTrait {} 28 | -------------------------------------------------------------------------------- /tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsParquetColumnIndexSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "330"} 19 | spark-rapids-shim-json-lines ***/ 20 | package org.apache.spark.sql.rapids.suites 21 | 22 | import org.apache.spark.sql.execution.datasources.parquet.ParquetColumnIndexSuite 23 | import org.apache.spark.sql.rapids.utils.RapidsSQLTestsBaseTrait 24 | 25 | class RapidsParquetColumnIndexSuite extends ParquetColumnIndexSuite with RapidsSQLTestsBaseTrait { 26 | } 27 | -------------------------------------------------------------------------------- /tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsParquetDeltaByteArrayEncodingSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "330"} 19 | spark-rapids-shim-json-lines ***/ 20 | package org.apache.spark.sql.rapids.suites 21 | 22 | import org.apache.spark.sql.execution.datasources.parquet.ParquetDeltaByteArrayEncodingSuite 23 | import org.apache.spark.sql.rapids.utils.RapidsSQLTestsBaseTrait 24 | 25 | class RapidsParquetDeltaByteArrayEncodingSuite 26 | extends ParquetDeltaByteArrayEncodingSuite 27 | with RapidsSQLTestsBaseTrait {} 28 | -------------------------------------------------------------------------------- /tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsParquetFieldIdIOSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "330"} 19 | spark-rapids-shim-json-lines ***/ 20 | package org.apache.spark.sql.rapids.suites 21 | 22 | import org.apache.spark.sql.execution.datasources.parquet.ParquetFieldIdIOSuite 23 | import org.apache.spark.sql.rapids.utils.RapidsSQLTestsBaseTrait 24 | 25 | class RapidsParquetFieldIdIOSuite extends ParquetFieldIdIOSuite with RapidsSQLTestsBaseTrait {} 26 | -------------------------------------------------------------------------------- /tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsParquetFieldIdSchemaSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "330"} 19 | spark-rapids-shim-json-lines ***/ 20 | package org.apache.spark.sql.rapids.suites 21 | 22 | import org.apache.spark.sql.execution.datasources.parquet.ParquetFieldIdSchemaSuite 23 | import org.apache.spark.sql.rapids.utils.RapidsSQLTestsBaseTrait 24 | 25 | class RapidsParquetFieldIdSchemaSuite 26 | extends ParquetFieldIdSchemaSuite 27 | with RapidsSQLTestsBaseTrait {} 28 | -------------------------------------------------------------------------------- /tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsParquetFileFormatSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "330"} 19 | spark-rapids-shim-json-lines ***/ 20 | package org.apache.spark.sql.rapids.suites 21 | 22 | import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormatSuite 23 | import org.apache.spark.sql.rapids.utils.RapidsSQLTestsBaseTrait 24 | 25 | class RapidsParquetFileFormatSuite 26 | extends ParquetFileFormatSuite 27 | with RapidsSQLTestsBaseTrait {} 28 | -------------------------------------------------------------------------------- /tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsParquetSchemaSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "330"} 19 | spark-rapids-shim-json-lines ***/ 20 | package org.apache.spark.sql.rapids.suites 21 | 22 | import org.apache.spark.sql.execution.datasources.parquet.ParquetSchemaSuite 23 | import org.apache.spark.sql.rapids.utils.RapidsSQLTestsBaseTrait 24 | 25 | class RapidsParquetSchemaSuite extends ParquetSchemaSuite with RapidsSQLTestsBaseTrait {} 26 | -------------------------------------------------------------------------------- /tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsParquetVectorizedSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "330"} 19 | spark-rapids-shim-json-lines ***/ 20 | package org.apache.spark.sql.rapids.suites 21 | 22 | import org.apache.spark.sql.execution.datasources.parquet.ParquetVectorizedSuite 23 | import org.apache.spark.sql.rapids.utils.RapidsSQLTestsBaseTrait 24 | 25 | class RapidsParquetVectorizedSuite extends ParquetVectorizedSuite with RapidsSQLTestsBaseTrait {} 26 | -------------------------------------------------------------------------------- /tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsRegexpExpressionsSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "330"} 19 | spark-rapids-shim-json-lines ***/ 20 | package org.apache.spark.sql.rapids.suites 21 | 22 | import org.apache.spark.sql.catalyst.expressions.RegexpExpressionsSuite 23 | import org.apache.spark.sql.rapids.utils.RapidsTestsTrait 24 | 25 | class RapidsRegexpExpressionsSuite extends RegexpExpressionsSuite with RapidsTestsTrait {} 26 | -------------------------------------------------------------------------------- /tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsStringExpressionsSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "330"} 19 | spark-rapids-shim-json-lines ***/ 20 | package org.apache.spark.sql.rapids.suites 21 | 22 | import org.apache.spark.sql.catalyst.expressions.StringExpressionsSuite 23 | import org.apache.spark.sql.rapids.utils.RapidsTestsTrait 24 | 25 | class RapidsStringExpressionsSuite extends StringExpressionsSuite with RapidsTestsTrait {} 26 | -------------------------------------------------------------------------------- /tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsStringFunctionsSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024, NVIDIA CORPORATION. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /*** spark-rapids-shim-json-lines 18 | {"spark": "330"} 19 | spark-rapids-shim-json-lines ***/ 20 | package org.apache.spark.sql.rapids.suites 21 | 22 | import org.apache.spark.sql.StringFunctionsSuite 23 | import org.apache.spark.sql.rapids.utils.RapidsSQLTestsTrait 24 | 25 | class RapidsStringFunctionsSuite 26 | extends StringFunctionsSuite 27 | with RapidsSQLTestsTrait { 28 | } 29 | -------------------------------------------------------------------------------- /tools/.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | *.iml 3 | target 4 | event_log_profiling.log 5 | ## ignore qual-ui static files 6 | **/ui/assets/ 7 | **/ui-dependencies-cache/ 8 | **/ui/js/data-output.js 9 | ## ignore output folders of the test scripts 10 | **/dev/qualification-output/ 11 | -------------------------------------------------------------------------------- /tools/generated_files/320/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 14 | -------------------------------------------------------------------------------- /tools/generated_files/321/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 14 | -------------------------------------------------------------------------------- /tools/generated_files/321cdh/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 14 | -------------------------------------------------------------------------------- /tools/generated_files/322/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 14 | -------------------------------------------------------------------------------- /tools/generated_files/323/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 14 | -------------------------------------------------------------------------------- /tools/generated_files/324/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 14 | -------------------------------------------------------------------------------- /tools/generated_files/330/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 14 | -------------------------------------------------------------------------------- /tools/generated_files/330cdh/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 14 | -------------------------------------------------------------------------------- /tools/generated_files/331/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 14 | -------------------------------------------------------------------------------- /tools/generated_files/332/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 14 | -------------------------------------------------------------------------------- /tools/generated_files/332cdh/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 14 | -------------------------------------------------------------------------------- /tools/generated_files/333/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 14 | -------------------------------------------------------------------------------- /tools/generated_files/334/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 14 | -------------------------------------------------------------------------------- /tools/generated_files/340/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 14 | -------------------------------------------------------------------------------- /tools/generated_files/341/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 14 | -------------------------------------------------------------------------------- /tools/generated_files/342/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 14 | -------------------------------------------------------------------------------- /tools/generated_files/343/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 14 | -------------------------------------------------------------------------------- /tools/generated_files/344/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 14 | -------------------------------------------------------------------------------- /tools/generated_files/350/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 14 | -------------------------------------------------------------------------------- /tools/generated_files/350db143/supportedDataSource.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/spark-rapids/6043a54df48aa74d8dd1ef4aa13c76c34c37b0e8/tools/generated_files/350db143/supportedDataSource.csv -------------------------------------------------------------------------------- /tools/generated_files/351/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 14 | -------------------------------------------------------------------------------- /tools/generated_files/352/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 14 | -------------------------------------------------------------------------------- /tools/generated_files/353/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 14 | -------------------------------------------------------------------------------- /tools/generated_files/354/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 14 | -------------------------------------------------------------------------------- /tools/generated_files/355/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 14 | -------------------------------------------------------------------------------- /tools/generated_files/356/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 14 | -------------------------------------------------------------------------------- /tools/generated_files/400/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S 14 | -------------------------------------------------------------------------------- /tools/generated_files/supportedDataSource.csv: -------------------------------------------------------------------------------- 1 | Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH 2 | Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO 3 | CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA 4 | Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 5 | Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 6 | HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 7 | HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS 8 | Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 9 | JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA 10 | ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 11 | ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA 12 | Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 13 | Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA 14 | -------------------------------------------------------------------------------- /udf-compiler/README.md: -------------------------------------------------------------------------------- 1 | UDF Compiler 2 | ============ 3 | 4 | How to run tests 5 | ---------------- 6 | 7 | From `rapids-plugin-4-spark` root directory, use this command to run the `OpcodeSuite`: 8 | 9 | ``` 10 | mvn test -DwildcardSuites=com.nvidia.spark.OpcodeSuite 11 | ``` 12 | 13 | How to run 14 | ---------- 15 | 16 | The UDF compiler is included in the rapids-4-spark jar that is produced by the `dist` maven project. Set up your cluster to run the RAPIDS Accelerator for Apache Spark 17 | and this udf plugin will be automatically injected to spark extensions when `com.nvidia.spark.SQLPlugin` is set. 18 | 19 | The plugin is still disabled by default and you will need to set `spark.rapids.sql.udfCompiler.enabled` to `true` to enable it. 20 | --------------------------------------------------------------------------------