├── .java-version ├── core └── src │ ├── test │ ├── resources │ │ ├── org │ │ │ └── apache │ │ │ │ └── iceberg │ │ │ │ └── puffin │ │ │ │ └── v1 │ │ │ │ ├── empty-puffin-uncompressed.bin │ │ │ │ ├── sample-metric-data-uncompressed.bin │ │ │ │ └── sample-metric-data-compressed-zstd.bin │ │ ├── TableMetadataUnsupportedVersion.json │ │ ├── TableMetadataV1MissingSchemaType.json │ │ ├── TableMetadataV1Valid.json │ │ └── TableMetadataV2MissingSortOrder.json │ └── java │ │ └── org │ │ └── apache │ │ └── iceberg │ │ ├── V2TableTestBase.java │ │ ├── TestEnvironmentContext.java │ │ └── rest │ │ └── auth │ │ └── AuthSessionUtil.java │ └── main │ └── java │ └── org │ └── apache │ └── iceberg │ ├── rest │ ├── RESTRequest.java │ ├── RESTResponse.java │ ├── ErrorHandler.java │ └── RESTMessage.java │ ├── HasTableOperations.java │ ├── PositionDeletesScanTask.java │ ├── actions │ ├── SnapshotUpdateAction.java │ ├── BaseMigrateTableActionResult.java │ ├── BaseSnapshotTableActionResult.java │ └── BaseDeleteOrphanFilesActionResult.java │ ├── hadoop │ ├── Configurable.java │ └── ConfigProperties.java │ ├── view │ └── UnknownViewRepresentation.java │ ├── util │ ├── SerializableSupplier.java │ └── LocationUtil.java │ ├── avro │ ├── ValueReader.java │ ├── SupportsRowPosition.java │ ├── ValueWriter.java │ └── MetricsAwareDatumWriter.java │ ├── InheritableMetadata.java │ ├── ClientPool.java │ ├── puffin │ ├── StandardPuffinProperties.java │ └── StandardBlobTypes.java │ ├── deletes │ └── DeleteCounter.java │ ├── expressions │ └── Zorder.java │ ├── SplitScanTaskIterator.java │ ├── encryption │ └── EncryptionKeyMetadatas.java │ ├── jdbc │ ├── UncheckedSQLException.java │ └── UncheckedInterruptedException.java │ └── metrics │ └── CommitReport.java ├── bundled-guava └── NOTICE ├── gradle └── wrapper │ └── gradle-wrapper.properties ├── dev └── .rat-excludes ├── spark ├── v2.4 │ └── spark │ │ └── src │ │ ├── test │ │ └── resources │ │ │ └── data │ │ │ ├── books.json │ │ │ └── new-books.json │ │ ├── main │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ └── org.apache.spark.sql.sources.DataSourceRegister │ │ └── jmh │ │ └── java │ │ └── org │ │ └── apache │ │ └── iceberg │ │ └── spark │ │ └── source │ │ └── Action.java ├── v3.1 │ ├── spark │ │ └── src │ │ │ ├── main │ │ │ ├── resources │ │ │ │ └── META-INF │ │ │ │ │ └── services │ │ │ │ │ └── org.apache.spark.sql.sources.DataSourceRegister │ │ │ └── java │ │ │ │ └── org │ │ │ │ └── apache │ │ │ │ └── spark │ │ │ │ └── sql │ │ │ │ └── connector │ │ │ │ └── iceberg │ │ │ │ └── distributions │ │ │ │ ├── impl │ │ │ │ └── UnspecifiedDistributionImpl.java │ │ │ │ ├── Distribution.java │ │ │ │ └── UnspecifiedDistribution.java │ │ │ └── jmh │ │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── iceberg │ │ │ └── spark │ │ │ └── source │ │ │ └── Action.java │ └── spark-extensions │ │ └── src │ │ └── main │ │ └── scala │ │ └── org │ │ └── apache │ │ └── spark │ │ └── sql │ │ └── catalyst │ │ └── plans │ │ └── logical │ │ └── BranchOptions.scala ├── v3.2 │ ├── spark │ │ └── src │ │ │ ├── main │ │ │ ├── resources │ │ │ │ └── META-INF │ │ │ │ │ └── services │ │ │ │ │ └── org.apache.spark.sql.sources.DataSourceRegister │ │ │ └── java │ │ │ │ └── org │ │ │ │ └── apache │ │ │ │ ├── spark │ │ │ │ └── sql │ │ │ │ │ └── connector │ │ │ │ │ └── iceberg │ │ │ │ │ └── write │ │ │ │ │ ├── RowLevelOperationBuilder.java │ │ │ │ │ ├── DeltaWriteBuilder.java │ │ │ │ │ ├── DeltaBatchWrite.java │ │ │ │ │ └── DeltaWriterFactory.java │ │ │ │ └── iceberg │ │ │ │ └── spark │ │ │ │ └── source │ │ │ │ ├── HasIcebergCatalog.java │ │ │ │ └── metrics │ │ │ │ ├── TaskNumSplits.java │ │ │ │ └── TaskNumDeletes.java │ │ │ └── jmh │ │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── iceberg │ │ │ └── spark │ │ │ └── source │ │ │ └── Action.java │ └── spark-extensions │ │ └── src │ │ └── main │ │ └── scala │ │ └── org │ │ └── apache │ │ └── spark │ │ └── sql │ │ ├── catalyst │ │ ├── plans │ │ │ └── logical │ │ │ │ ├── BranchOptions.scala │ │ │ │ ├── RowLevelCommand.scala │ │ │ │ └── NoStatsUnaryNode.scala │ │ └── util │ │ │ ├── RowDeltaUtils.scala │ │ │ └── WriteDeltaProjections.scala │ │ └── connector │ │ └── write │ │ └── RowLevelOperationInfoImpl.scala └── v3.3 │ ├── spark │ └── src │ │ ├── main │ │ ├── resources │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── org.apache.spark.sql.sources.DataSourceRegister │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ ├── iceberg │ │ │ └── spark │ │ │ │ └── source │ │ │ │ ├── HasIcebergCatalog.java │ │ │ │ └── metrics │ │ │ │ ├── TaskNumSplits.java │ │ │ │ └── TaskNumDeletes.java │ │ │ └── spark │ │ │ └── sql │ │ │ └── connector │ │ │ └── iceberg │ │ │ └── write │ │ │ ├── DeltaWriteBuilder.java │ │ │ ├── DeltaBatchWrite.java │ │ │ └── DeltaWriterFactory.java │ │ ├── jmh │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── iceberg │ │ │ └── spark │ │ │ └── source │ │ │ └── Action.java │ │ └── test │ │ └── java │ │ └── org │ │ └── apache │ │ └── iceberg │ │ └── spark │ │ └── sql │ │ ├── TestPartitionedWrites.java │ │ └── TestUnpartitionedWrites.java │ └── spark-extensions │ └── src │ └── main │ └── scala │ └── org │ └── apache │ └── spark │ └── sql │ └── catalyst │ ├── plans │ └── logical │ │ ├── TagOptions.scala │ │ ├── BranchOptions.scala │ │ ├── RowLevelCommand.scala │ │ ├── NoStatsUnaryNode.scala │ │ └── DropTag.scala │ └── util │ ├── RowDeltaUtils.scala │ └── WriteDeltaProjections.scala ├── .github ├── ISSUE_TEMPLATE │ ├── iceberg_question.yml │ └── iceberg_improvement.yml ├── workflows │ ├── license_check.yml │ └── labeler.yml └── dependabot.yml ├── .baseline └── copyright │ ├── 001_apache-2.0.txt │ └── copyright-header-java.txt ├── flink ├── v1.14 │ ├── flink │ │ └── src │ │ │ ├── main │ │ │ ├── resources │ │ │ │ └── META-INF │ │ │ │ │ └── services │ │ │ │ │ ├── org.apache.flink.table.factories.Factory │ │ │ │ │ └── org.apache.flink.table.factories.TableFactory │ │ │ └── java │ │ │ │ └── org │ │ │ │ └── apache │ │ │ │ └── iceberg │ │ │ │ └── flink │ │ │ │ ├── source │ │ │ │ ├── split │ │ │ │ │ └── IcebergSourceSplitStatus.java │ │ │ │ └── assigner │ │ │ │ │ ├── SplitAssignerType.java │ │ │ │ │ └── SplitAssignerFactory.java │ │ │ │ ├── FlinkEnvironmentContext.java │ │ │ │ └── util │ │ │ │ └── FlinkPackage.java │ │ │ └── test │ │ │ ├── resources │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── org.apache.flink.table.factories.Factory │ │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── iceberg │ │ │ └── flink │ │ │ └── util │ │ │ └── TestFlinkPackage.java │ └── flink-runtime │ │ └── src │ │ └── integration │ │ └── java │ │ └── org │ │ └── apache │ │ └── iceberg │ │ └── flink │ │ └── IcebergConnectorSmokeTest.java ├── v1.15 │ ├── flink │ │ └── src │ │ │ ├── main │ │ │ ├── resources │ │ │ │ └── META-INF │ │ │ │ │ └── services │ │ │ │ │ ├── org.apache.flink.table.factories.Factory │ │ │ │ │ └── org.apache.flink.table.factories.TableFactory │ │ │ └── java │ │ │ │ └── org │ │ │ │ └── apache │ │ │ │ └── iceberg │ │ │ │ └── flink │ │ │ │ ├── source │ │ │ │ ├── split │ │ │ │ │ └── IcebergSourceSplitStatus.java │ │ │ │ └── assigner │ │ │ │ │ ├── SplitAssignerType.java │ │ │ │ │ └── SplitAssignerFactory.java │ │ │ │ ├── FlinkEnvironmentContext.java │ │ │ │ └── util │ │ │ │ └── FlinkPackage.java │ │ │ └── test │ │ │ ├── resources │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── org.apache.flink.table.factories.Factory │ │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── iceberg │ │ │ └── flink │ │ │ └── util │ │ │ └── TestFlinkPackage.java │ └── flink-runtime │ │ └── src │ │ └── integration │ │ └── java │ │ └── org │ │ └── apache │ │ └── iceberg │ │ └── flink │ │ └── IcebergConnectorSmokeTest.java ├── v1.16 │ ├── flink │ │ └── src │ │ │ ├── main │ │ │ ├── resources │ │ │ │ └── META-INF │ │ │ │ │ └── services │ │ │ │ │ ├── org.apache.flink.table.factories.Factory │ │ │ │ │ └── org.apache.flink.table.factories.TableFactory │ │ │ └── java │ │ │ │ └── org │ │ │ │ └── apache │ │ │ │ └── iceberg │ │ │ │ └── flink │ │ │ │ ├── source │ │ │ │ ├── split │ │ │ │ │ └── IcebergSourceSplitStatus.java │ │ │ │ └── assigner │ │ │ │ │ ├── SplitAssignerType.java │ │ │ │ │ └── SplitAssignerFactory.java │ │ │ │ ├── FlinkEnvironmentContext.java │ │ │ │ └── util │ │ │ │ └── FlinkPackage.java │ │ │ └── test │ │ │ ├── resources │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── org.apache.flink.table.factories.Factory │ │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── iceberg │ │ │ └── flink │ │ │ └── util │ │ │ └── TestFlinkPackage.java │ └── flink-runtime │ │ └── src │ │ └── integration │ │ └── java │ │ └── org │ │ └── apache │ │ └── iceberg │ │ └── flink │ │ └── IcebergConnectorSmokeTest.java └── build.gradle ├── api └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── iceberg │ │ ├── metrics │ │ └── MetricsReport.java │ │ ├── io │ │ ├── DelegatingInputStream.java │ │ ├── DelegatingOutputStream.java │ │ ├── SupportsBulkOperations.java │ │ ├── PositionOutputStream.java │ │ ├── BulkDeletionFailureException.java │ │ └── FileIOMetricsContext.java │ │ ├── events │ │ └── Listener.java │ │ ├── expressions │ │ ├── Term.java │ │ ├── UnboundTerm.java │ │ └── Reference.java │ │ ├── ChangelogOperation.java │ │ ├── Accessor.java │ │ ├── view │ │ ├── ViewRepresentation.java │ │ └── ViewHistoryEntry.java │ │ ├── IncrementalAppendScan.java │ │ ├── IncrementalChangelogScan.java │ │ ├── PartitionScanTask.java │ │ ├── StructLike.java │ │ ├── UpdateLocation.java │ │ ├── FileContent.java │ │ ├── NullOrder.java │ │ ├── util │ │ ├── SerializableFunction.java │ │ └── NaNUtil.java │ │ ├── HistoryEntry.java │ │ ├── exceptions │ │ ├── DuplicateWAPCommitException.java │ │ ├── RESTException.java │ │ ├── ServiceFailureException.java │ │ └── BadRequestException.java │ │ └── DataTask.java │ └── test │ └── java │ └── org │ └── apache │ └── iceberg │ └── TestTransformSerialization.java ├── parquet └── src │ └── main │ └── java │ └── org │ └── apache │ └── iceberg │ └── parquet │ ├── ParquetAvroReader.java │ └── ParquetValueReader.java ├── NOTICE ├── mr └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── iceberg │ │ └── mr │ │ ├── mapreduce │ │ └── IcebergSplitContainer.java │ │ └── hive │ │ └── serde │ │ └── objectinspector │ │ └── WriteObjectInspector.java │ └── test │ └── resources │ └── log4j.properties ├── .gitignore ├── version.properties ├── hive-metastore └── src │ └── main │ └── java │ └── org │ └── apache │ └── iceberg │ └── hive │ ├── HiveLock.java │ └── LockException.java ├── aws └── src │ ├── integration │ └── java │ │ └── org │ │ └── apache │ │ └── iceberg │ │ └── aws │ │ └── s3 │ │ └── S3TestUtil.java │ └── main │ └── java │ └── org │ └── apache │ └── iceberg │ └── aws │ └── s3 │ └── signer │ ├── S3SignResponse.java │ └── S3SignRequest.java ├── orc └── src │ └── main │ └── java │ └── org │ └── apache │ └── iceberg │ └── orc │ ├── OrcRowReader.java │ ├── OrcBatchReader.java │ └── OrcValueReader.java ├── arrow └── src │ └── main │ └── java │ └── org │ └── apache │ └── iceberg │ └── arrow │ └── ArrowAllocation.java ├── gradle.properties ├── delta-lake └── src │ └── main │ └── java │ └── org │ └── apache │ └── iceberg │ └── delta │ └── BaseSnapshotDeltaLakeTableActionResult.java └── aliyun └── src └── test └── java └── org └── apache └── iceberg └── aliyun └── oss └── mock └── Range.java /.java-version: -------------------------------------------------------------------------------- 1 | 1.8 2 | -------------------------------------------------------------------------------- /core/src/test/resources/org/apache/iceberg/puffin/v1/empty-puffin-uncompressed.bin: -------------------------------------------------------------------------------- 1 | PFA1PFA1{"blobs":[]} PFA1 -------------------------------------------------------------------------------- /bundled-guava/NOTICE: -------------------------------------------------------------------------------- 1 | 2 | Apache Iceberg 3 | Copyright 2017-2022 The Apache Software Foundation 4 | 5 | This product includes software developed at 6 | The Apache Software Foundation (http://www.apache.org/). 7 | 8 | -------------------------------------------------------------------------------- /core/src/test/resources/org/apache/iceberg/puffin/v1/sample-metric-data-uncompressed.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linkedin/iceberg/HEAD/core/src/test/resources/org/apache/iceberg/puffin/v1/sample-metric-data-uncompressed.bin -------------------------------------------------------------------------------- /core/src/test/resources/org/apache/iceberg/puffin/v1/sample-metric-data-compressed-zstd.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linkedin/iceberg/HEAD/core/src/test/resources/org/apache/iceberg/puffin/v1/sample-metric-data-compressed-zstd.bin -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | # checksum was taken from https://gradle.org/release-checksums 4 | distributionSha256Sum=ff7bf6a86f09b9b2c40bb8f48b25fc19cf2b2664fd1d220cd7ab833ec758d0d7 5 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.0.2-bin.zip 6 | networkTimeout=10000 7 | zipStoreBase=GRADLE_USER_HOME 8 | zipStorePath=wrapper/dists 9 | -------------------------------------------------------------------------------- /dev/.rat-excludes: -------------------------------------------------------------------------------- 1 | version.txt 2 | versions.lock 3 | versions.props 4 | iceberg-build.properties 5 | .java-version 6 | books.json 7 | new-books.json 8 | build 9 | .gitignore 10 | .rat-excludes 11 | LICENSE 12 | NOTICE 13 | revapi.yml 14 | gradlew 15 | examples/* 16 | gradle/* 17 | .*\.sql 18 | .*\.iml 19 | .*\.iws 20 | .*\.html 21 | .*\.css 22 | .*\.js 23 | .*\.svg 24 | .*\.lock 25 | .*\.json 26 | .*\.bin 27 | package-list 28 | sitemap.xml 29 | derby.log 30 | .python-version 31 | .*_index.md 32 | -------------------------------------------------------------------------------- /spark/v2.4/spark/src/test/resources/data/books.json: -------------------------------------------------------------------------------- 1 | {"title":"Gone", "price":12, "author": "Michael Grant", "published": 1541776051, "genre": "fiction"} 2 | {"title":"Carry On", "price":10, "author": "Rainbow Rowell", "published": 1536505651, "genre": "fiction"} 3 | {"title":"Warward Son", "price":12, "author": "Rainbow Rowell", "published": 1504969651, "genre": "fiction"} 4 | {"title":"Heroes", "price":8, "author": "Stephen Fry", "published": 1504969651, "genre": "fiction"} 5 | {"title":"Vietnam", "price":15, "author": "Max Hastings", "genre": "non-fiction"} 6 | 7 | -------------------------------------------------------------------------------- /spark/v2.4/spark/src/test/resources/data/new-books.json: -------------------------------------------------------------------------------- 1 | {"title":"Harry Potter", "price":12, "author": "JK Rowling", "published": 1570719361, "genre": "fiction", "publisher": "ACME Books"} 2 | {"title":"Percy Jackson", "price":10, "author": "Rick Riordan", "published": 1547132161, "genre": "fiction", "publisher": "ACME Books"} 3 | {"title":"Cookie", "price":8, "author": "Jacqueline Wilson", "published": 1552229761, "genre": "fiction", "publisher": "ACME Books"} 4 | {"title":"Fangirl", "price":12, "author": "Rainbow Rowell", "published": 1552229761, "genre": "fiction", "publisher": "ACME Books"} 5 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/iceberg_question.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Iceberg Question 3 | description: Questions around Apache Iceberg 4 | labels: ["kind:question"] 5 | body: 6 | - type: markdown 7 | attributes: 8 | value: "Feel free to ask your question on [Slack](https://join.slack.com/t/apache-iceberg/shared_invite/zt-1oj35f7yc-wuTEhvkiqjGLje83B7rG8A) as well." 9 | - type: textarea 10 | attributes: 11 | label: Query engine 12 | description: Which query engine (Flink, Spark, EMR, etc) are you using? 13 | - type: textarea 14 | attributes: 15 | label: Question 16 | description: What is your question? 17 | validations: 18 | required: true 19 | -------------------------------------------------------------------------------- /.baseline/copyright/001_apache-2.0.txt: -------------------------------------------------------------------------------- 1 | Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | -------------------------------------------------------------------------------- /core/src/test/resources/TableMetadataUnsupportedVersion.json: -------------------------------------------------------------------------------- 1 | { 2 | "format-version": 3, 3 | "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", 4 | "location": "s3://bucket/test/location", 5 | "last-updated-ms": 1602638573874, 6 | "last-sequence-number": 0, 7 | "last-column-id": 3, 8 | "schema": { 9 | "type": "struct", 10 | "fields": [ 11 | { 12 | "id": 1, 13 | "name": "x", 14 | "required": true, 15 | "type": "long" 16 | }, 17 | { 18 | "id": 2, 19 | "name": "y", 20 | "required": true, 21 | "type": "long", 22 | "doc": "comment" 23 | }, 24 | { 25 | "id": 3, 26 | "name": "z", 27 | "required": true, 28 | "type": "long" 29 | } 30 | ] 31 | }, 32 | "partition-spec": [], 33 | "properties": {}, 34 | "current-snapshot-id": -1, 35 | "snapshots": [] 36 | } -------------------------------------------------------------------------------- /.baseline/copyright/copyright-header-java.txt: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/iceberg_improvement.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Iceberg Improvement / Feature Request 3 | description: New features with Apache Iceberg 4 | labels: ["kind:feature request"] 5 | body: 6 | - type: textarea 7 | attributes: 8 | label: Feature Request / Improvement 9 | description: Please describe the feature and elaborate on the use case and motivation behind it 10 | validations: 11 | required: true 12 | - type: dropdown 13 | attributes: 14 | label: Query engine 15 | description: Which query engine? 16 | multiple: false 17 | options: 18 | - Spark 19 | - Trino 20 | - Starburst 21 | - Snowflake 22 | - Dremio 23 | - Starrocks 24 | - Doris 25 | - EMR 26 | - Athena 27 | - PrestoDB 28 | - Flink 29 | - Impala 30 | - Hive 31 | - Other 32 | validations: 33 | required: false -------------------------------------------------------------------------------- /flink/v1.14/flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | org.apache.iceberg.flink.FlinkDynamicTableFactory 17 | -------------------------------------------------------------------------------- /flink/v1.14/flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.TableFactory: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | org.apache.iceberg.flink.FlinkCatalogFactory 17 | -------------------------------------------------------------------------------- /flink/v1.15/flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | org.apache.iceberg.flink.FlinkDynamicTableFactory 17 | -------------------------------------------------------------------------------- /flink/v1.15/flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.TableFactory: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | org.apache.iceberg.flink.FlinkCatalogFactory 17 | -------------------------------------------------------------------------------- /flink/v1.16/flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | org.apache.iceberg.flink.FlinkDynamicTableFactory 17 | -------------------------------------------------------------------------------- /flink/v1.16/flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.TableFactory: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | org.apache.iceberg.flink.FlinkCatalogFactory 17 | -------------------------------------------------------------------------------- /flink/v1.14/flink/src/test/resources/META-INF/services/org.apache.flink.table.factories.Factory: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | org.apache.iceberg.flink.source.BoundedTableFactory 17 | -------------------------------------------------------------------------------- /flink/v1.15/flink/src/test/resources/META-INF/services/org.apache.flink.table.factories.Factory: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | org.apache.iceberg.flink.source.BoundedTableFactory 17 | -------------------------------------------------------------------------------- /flink/v1.16/flink/src/test/resources/META-INF/services/org.apache.flink.table.factories.Factory: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | org.apache.iceberg.flink.source.BoundedTableFactory 17 | -------------------------------------------------------------------------------- /core/src/test/resources/TableMetadataV1MissingSchemaType.json: -------------------------------------------------------------------------------- 1 | { 2 | "format-version": 1, 3 | "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", 4 | "location": "s3://bucket/test/location", 5 | "last-updated-ms": 1602638573874, 6 | "last-column-id": 3, 7 | "schema": { 8 | "fields": [ 9 | { 10 | "id": 1, 11 | "name": "x", 12 | "required": true, 13 | "type": "long" 14 | }, 15 | { 16 | "id": 2, 17 | "name": "y", 18 | "required": true, 19 | "type": "long", 20 | "doc": "comment" 21 | }, 22 | { 23 | "id": 3, 24 | "name": "z", 25 | "required": true, 26 | "type": "long" 27 | } 28 | ] 29 | }, 30 | "partition-spec": [ 31 | { 32 | "name": "x", 33 | "transform": "identity", 34 | "source-id": 1, 35 | "field-id": 1000 36 | } 37 | ], 38 | "properties": {}, 39 | "current-snapshot-id": -1, 40 | "snapshots": [] 41 | } -------------------------------------------------------------------------------- /spark/v2.4/spark/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | org.apache.iceberg.spark.source.IcebergSource 21 | -------------------------------------------------------------------------------- /spark/v3.1/spark/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | org.apache.iceberg.spark.source.IcebergSource 21 | -------------------------------------------------------------------------------- /spark/v3.2/spark/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | org.apache.iceberg.spark.source.IcebergSource 21 | -------------------------------------------------------------------------------- /spark/v3.3/spark/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | org.apache.iceberg.spark.source.IcebergSource 21 | -------------------------------------------------------------------------------- /core/src/test/resources/TableMetadataV1Valid.json: -------------------------------------------------------------------------------- 1 | { 2 | "format-version": 1, 3 | "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", 4 | "location": "s3://bucket/test/location", 5 | "last-updated-ms": 1602638573874, 6 | "last-column-id": 3, 7 | "schema": { 8 | "type": "struct", 9 | "fields": [ 10 | { 11 | "id": 1, 12 | "name": "x", 13 | "required": true, 14 | "type": "long" 15 | }, 16 | { 17 | "id": 2, 18 | "name": "y", 19 | "required": true, 20 | "type": "long", 21 | "doc": "comment" 22 | }, 23 | { 24 | "id": 3, 25 | "name": "z", 26 | "required": true, 27 | "type": "long" 28 | } 29 | ] 30 | }, 31 | "partition-spec": [ 32 | { 33 | "name": "x", 34 | "transform": "identity", 35 | "source-id": 1, 36 | "field-id": 1000 37 | } 38 | ], 39 | "properties": {}, 40 | "current-snapshot-id": -1, 41 | "snapshots": [] 42 | } -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/metrics/MetricsReport.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.metrics; 20 | 21 | public interface MetricsReport {} 22 | -------------------------------------------------------------------------------- /parquet/src/main/java/org/apache/iceberg/parquet/ParquetAvroReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.parquet; 20 | 21 | public class ParquetAvroReader {} 22 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | 2 | Apache Iceberg 3 | Copyright 2017-2022 The Apache Software Foundation 4 | 5 | This product includes software developed at 6 | The Apache Software Foundation (http://www.apache.org/). 7 | 8 | -------------------------------------------------------------------------------- 9 | 10 | This project includes code from Kite, developed at Cloudera, Inc. with 11 | the following copyright notice: 12 | 13 | | Copyright 2013 Cloudera Inc. 14 | | 15 | | Licensed under the Apache License, Version 2.0 (the "License"); 16 | | you may not use this file except in compliance with the License. 17 | | You may obtain a copy of the License at 18 | | 19 | | http://www.apache.org/licenses/LICENSE-2.0 20 | | 21 | | Unless required by applicable law or agreed to in writing, software 22 | | distributed under the License is distributed on an "AS IS" BASIS, 23 | | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 24 | | See the License for the specific language governing permissions and 25 | | limitations under the License. 26 | 27 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/rest/RESTRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.rest; 20 | 21 | /** Interface to mark a REST request. */ 22 | public interface RESTRequest extends RESTMessage {} 23 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/rest/RESTResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.rest; 20 | 21 | /** Interface to mark a REST response */ 22 | public interface RESTResponse extends RESTMessage {} 23 | -------------------------------------------------------------------------------- /spark/v2.4/spark/src/jmh/java/org/apache/iceberg/spark/source/Action.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.spark.source; 20 | 21 | @FunctionalInterface 22 | public interface Action { 23 | void invoke(); 24 | } 25 | -------------------------------------------------------------------------------- /spark/v3.1/spark/src/jmh/java/org/apache/iceberg/spark/source/Action.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.spark.source; 20 | 21 | @FunctionalInterface 22 | public interface Action { 23 | void invoke(); 24 | } 25 | -------------------------------------------------------------------------------- /spark/v3.2/spark/src/jmh/java/org/apache/iceberg/spark/source/Action.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.spark.source; 20 | 21 | @FunctionalInterface 22 | public interface Action { 23 | void invoke(); 24 | } 25 | -------------------------------------------------------------------------------- /spark/v3.3/spark/src/jmh/java/org/apache/iceberg/spark/source/Action.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.spark.source; 20 | 21 | @FunctionalInterface 22 | public interface Action { 23 | void invoke(); 24 | } 25 | -------------------------------------------------------------------------------- /core/src/test/java/org/apache/iceberg/V2TableTestBase.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg; 20 | 21 | public class V2TableTestBase extends TableTestBase { 22 | public V2TableTestBase() { 23 | super(2); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergSplitContainer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.mr.mapreduce; 20 | 21 | public interface IcebergSplitContainer { 22 | 23 | IcebergSplit icebergSplit(); 24 | } 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | .DS_Store 3 | .cache 4 | tmp/ 5 | site 6 | 7 | # intellij files 8 | .idea 9 | .idea_modules/ 10 | *.ipr 11 | *.iws 12 | *.iml 13 | out 14 | 15 | # gradle build 16 | .gradle 17 | .out/ 18 | build 19 | dependencies.lock 20 | **/dependencies.lock 21 | gradle/ 22 | gradle/wrapper/gradle-wrapper.jar 23 | 24 | # rat library install location 25 | lib/ 26 | 27 | # web site build 28 | site/site 29 | 30 | # benchmark output 31 | spark/v2.4/spark/benchmark/* 32 | spark/v3.1/spark/benchmark/* 33 | spark/v3.2/spark/benchmark/* 34 | spark/v3.3/spark/benchmark/* 35 | data/benchmark/* 36 | 37 | __pycache__/ 38 | *.py[cod] 39 | .eggs/ 40 | .tox/ 41 | env/ 42 | venv/ 43 | *.egg-info/ 44 | test-reports 45 | build/ 46 | dist/ 47 | sdist/ 48 | .coverage 49 | coverage.xml 50 | .pytest_cache/ 51 | 52 | # vscode/eclipse files 53 | .classpath 54 | .project 55 | .settings 56 | bin/ 57 | 58 | # Hive/metastore files 59 | metastore_db/ 60 | 61 | # Spark/metastore files 62 | spark-warehouse/ 63 | derby.log 64 | 65 | # Python stuff 66 | python/.mypy_cache/ 67 | python/htmlcov 68 | python/coverage.xml 69 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/io/DelegatingInputStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.io; 20 | 21 | import java.io.InputStream; 22 | 23 | public interface DelegatingInputStream { 24 | InputStream getDelegate(); 25 | } 26 | -------------------------------------------------------------------------------- /.github/workflows/license_check.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | name: "Run License Check" 21 | on: pull_request 22 | 23 | jobs: 24 | rat: 25 | runs-on: ubuntu-22.04 26 | steps: 27 | - uses: actions/checkout@v4 28 | - run: | 29 | dev/check-license 30 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/io/DelegatingOutputStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.io; 20 | 21 | import java.io.OutputStream; 22 | 23 | public interface DelegatingOutputStream { 24 | OutputStream getDelegate(); 25 | } 26 | -------------------------------------------------------------------------------- /version.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | # 18 | 19 | # Version of the produced binaries. 20 | # The version is inferred by shipkit-auto-version Gradle plugin (https://github.com/shipkit/shipkit-auto-version) 21 | # Using 900+ series to indicate pre-release versions 22 | version=1.2.0.901 23 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/events/Listener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.events; 20 | 21 | /** A listener interface that can receive notifications. */ 22 | public interface Listener { 23 | void notify(E event); 24 | } 25 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/HasTableOperations.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg; 20 | 21 | /** Used to expose a table's TableOperations. */ 22 | public interface HasTableOperations { 23 | TableOperations operations(); 24 | } 25 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/PositionDeletesScanTask.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg; 20 | 21 | /** A {@link ScanTask} for position delete files */ 22 | public interface PositionDeletesScanTask extends ContentScanTask {} 23 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/expressions/Term.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.expressions; 20 | 21 | import java.io.Serializable; 22 | 23 | /** An expression that evaluates to a value. */ 24 | public interface Term extends Serializable {} 25 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/actions/SnapshotUpdateAction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.actions; 20 | 21 | public interface SnapshotUpdateAction extends Action { 22 | ThisT set(String property, String value); 23 | } 24 | -------------------------------------------------------------------------------- /spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TagOptions.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.spark.sql.catalyst.plans.logical 21 | 22 | case class TagOptions(snapshotId: Option[Long], snapshotRefRetain: Option[Long]) 23 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/hadoop/Configurable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.hadoop; 20 | 21 | /** Interface used to avoid runtime dependencies on Hadoop Configurable */ 22 | public interface Configurable { 23 | void setConf(C conf); 24 | } 25 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/view/UnknownViewRepresentation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.view; 20 | 21 | import org.immutables.value.Value; 22 | 23 | @Value.Immutable 24 | public interface UnknownViewRepresentation extends ViewRepresentation {} 25 | -------------------------------------------------------------------------------- /flink/v1.14/flink/src/main/java/org/apache/iceberg/flink/source/split/IcebergSourceSplitStatus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.flink.source.split; 20 | 21 | public enum IcebergSourceSplitStatus { 22 | UNASSIGNED, 23 | ASSIGNED, 24 | COMPLETED 25 | } 26 | -------------------------------------------------------------------------------- /flink/v1.15/flink/src/main/java/org/apache/iceberg/flink/source/split/IcebergSourceSplitStatus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.flink.source.split; 20 | 21 | public enum IcebergSourceSplitStatus { 22 | UNASSIGNED, 23 | ASSIGNED, 24 | COMPLETED 25 | } 26 | -------------------------------------------------------------------------------- /flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/source/split/IcebergSourceSplitStatus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.flink.source.split; 20 | 21 | public enum IcebergSourceSplitStatus { 22 | UNASSIGNED, 23 | ASSIGNED, 24 | COMPLETED 25 | } 26 | -------------------------------------------------------------------------------- /hive-metastore/src/main/java/org/apache/iceberg/hive/HiveLock.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.hive; 20 | 21 | interface HiveLock { 22 | void lock() throws LockException; 23 | 24 | void ensureActive() throws LockException; 25 | 26 | void unlock(); 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/util/SerializableSupplier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.util; 20 | 21 | import java.io.Serializable; 22 | import java.util.function.Supplier; 23 | 24 | public interface SerializableSupplier extends Supplier, Serializable {} 25 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/ChangelogOperation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg; 20 | 21 | /** An enum representing possible operations in a changelog. */ 22 | public enum ChangelogOperation { 23 | INSERT, 24 | DELETE, 25 | UPDATE_BEFORE, 26 | UPDATE_AFTER 27 | } 28 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/Accessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg; 20 | 21 | import java.io.Serializable; 22 | import org.apache.iceberg.types.Type; 23 | 24 | public interface Accessor extends Serializable { 25 | Object get(T container); 26 | 27 | Type type(); 28 | } 29 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/avro/ValueReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.avro; 20 | 21 | import java.io.IOException; 22 | import org.apache.avro.io.Decoder; 23 | 24 | public interface ValueReader { 25 | T read(Decoder decoder, Object reuse) throws IOException; 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/InheritableMetadata.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg; 20 | 21 | import java.io.Serializable; 22 | 23 | interface InheritableMetadata extends Serializable { 24 | > ManifestEntry apply(ManifestEntry manifestEntry); 25 | } 26 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/view/ViewRepresentation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.view; 20 | 21 | public interface ViewRepresentation { 22 | 23 | class Type { 24 | private Type() {} 25 | 26 | public static final String SQL = "sql"; 27 | } 28 | 29 | String type(); 30 | } 31 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/IncrementalAppendScan.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg; 20 | 21 | /** API for configuring an incremental table scan for appends only snapshots */ 22 | public interface IncrementalAppendScan 23 | extends IncrementalScan {} 24 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/expressions/UnboundTerm.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.expressions; 20 | 21 | /** 22 | * Represents an unbound term. 23 | * 24 | * @param the Java type of values produced by this term 25 | */ 26 | public interface UnboundTerm extends Unbound>, Term {} 27 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/IncrementalChangelogScan.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg; 20 | 21 | /** API for configuring a scan for table changes. */ 22 | public interface IncrementalChangelogScan 23 | extends IncrementalScan< 24 | IncrementalChangelogScan, ChangelogScanTask, ScanTaskGroup> {} 25 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/expressions/Reference.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.expressions; 20 | 21 | /** 22 | * Represents a variable reference in an {@link Expression expression}. 23 | * 24 | * @see BoundReference 25 | * @see NamedReference 26 | */ 27 | public interface Reference extends Term { 28 | String name(); 29 | } 30 | -------------------------------------------------------------------------------- /spark/v3.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/BranchOptions.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.spark.sql.catalyst.plans.logical 21 | 22 | case class BranchOptions(snapshotId: Option[Long], numSnapshots: Option[Long], 23 | snapshotRetain: Option[Long], snapshotRefRetain: Option[Long]) 24 | -------------------------------------------------------------------------------- /spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/BranchOptions.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.spark.sql.catalyst.plans.logical 21 | 22 | case class BranchOptions(snapshotId: Option[Long], numSnapshots: Option[Long], 23 | snapshotRetain: Option[Long], snapshotRefRetain: Option[Long]) 24 | -------------------------------------------------------------------------------- /spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/BranchOptions.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.spark.sql.catalyst.plans.logical 21 | 22 | case class BranchOptions (snapshotId: Option[Long], numSnapshots: Option[Long], 23 | snapshotRetain: Option[Long], snapshotRefRetain: Option[Long]) 24 | -------------------------------------------------------------------------------- /.github/workflows/labeler.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | name: "Pull Request Labeler" 21 | on: pull_request_target 22 | 23 | permissions: 24 | contents: read 25 | pull-requests: write 26 | 27 | jobs: 28 | triage: 29 | runs-on: ubuntu-22.04 30 | steps: 31 | - uses: actions/labeler@v4 32 | with: 33 | repo-token: "${{ secrets.GITHUB_TOKEN }}" 34 | sync-labels: true 35 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/hadoop/ConfigProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.hadoop; 20 | 21 | public class ConfigProperties { 22 | 23 | private ConfigProperties() {} 24 | 25 | public static final String ENGINE_HIVE_ENABLED = "iceberg.engine.hive.enabled"; 26 | public static final String KEEP_HIVE_STATS = "iceberg.hive.keep.stats"; 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/rest/ErrorHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.rest; 20 | 21 | import java.util.function.Consumer; 22 | import org.apache.iceberg.rest.responses.ErrorResponse; 23 | 24 | public abstract class ErrorHandler implements Consumer { 25 | 26 | public abstract ErrorResponse parseResponse(int code, String json); 27 | } 28 | -------------------------------------------------------------------------------- /spark/v3.1/spark/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/UnspecifiedDistributionImpl.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.spark.sql.connector.iceberg.distributions.impl; 20 | 21 | import org.apache.spark.sql.connector.iceberg.distributions.UnspecifiedDistribution; 22 | 23 | public class UnspecifiedDistributionImpl implements UnspecifiedDistribution {} 24 | -------------------------------------------------------------------------------- /mr/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Configuration for modules which are using log4j v1 (e.g. Hive2, DataNucleus) 19 | 20 | log4j.rootLogger=WARN,stdout 21 | log4j.threshold=ALL 22 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 23 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 24 | log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n 25 | -------------------------------------------------------------------------------- /spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/util/RowDeltaUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.spark.sql.catalyst.util 21 | 22 | object RowDeltaUtils { 23 | final val OPERATION_COLUMN: String = "__row_operation" 24 | final val DELETE_OPERATION: Int = 1 25 | final val UPDATE_OPERATION: Int = 2 26 | final val INSERT_OPERATION: Int = 3 27 | } 28 | -------------------------------------------------------------------------------- /spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/util/RowDeltaUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.spark.sql.catalyst.util 21 | 22 | object RowDeltaUtils { 23 | final val OPERATION_COLUMN: String = "__row_operation" 24 | final val DELETE_OPERATION: Int = 1 25 | final val UPDATE_OPERATION: Int = 2 26 | final val INSERT_OPERATION: Int = 3 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/avro/SupportsRowPosition.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.avro; 20 | 21 | import java.util.function.Supplier; 22 | 23 | /** 24 | * Interface for readers that accept a callback to determine the starting row position of an Avro 25 | * split. 26 | */ 27 | public interface SupportsRowPosition { 28 | void setRowPositionSupplier(Supplier posSupplier); 29 | } 30 | -------------------------------------------------------------------------------- /aws/src/integration/java/org/apache/iceberg/aws/s3/S3TestUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.aws.s3; 20 | 21 | public class S3TestUtil { 22 | 23 | private S3TestUtil() {} 24 | 25 | public static String getBucketFromUri(String s3Uri) { 26 | return new S3URI(s3Uri).bucket(); 27 | } 28 | 29 | public static String getKeyFromUri(String s3Uri) { 30 | return new S3URI(s3Uri).key(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /orc/src/main/java/org/apache/iceberg/orc/OrcRowReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.orc; 20 | 21 | import org.apache.orc.storage.ql.exec.vector.VectorizedRowBatch; 22 | 23 | /** Used for implementing ORC row readers. */ 24 | public interface OrcRowReader { 25 | 26 | /** Reads a row. */ 27 | T read(VectorizedRowBatch batch, int row); 28 | 29 | void setBatchContext(long batchOffsetInFile); 30 | } 31 | -------------------------------------------------------------------------------- /spark/v3.1/spark/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/Distribution.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.spark.sql.connector.iceberg.distributions; 20 | 21 | import org.apache.spark.annotation.Experimental; 22 | 23 | /** 24 | * An interface that defines how data is distributed across partitions. 25 | * 26 | * @since 3.2.0 27 | */ 28 | @Experimental 29 | public interface Distribution {} 30 | -------------------------------------------------------------------------------- /orc/src/main/java/org/apache/iceberg/orc/OrcBatchReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.orc; 20 | 21 | import org.apache.orc.storage.ql.exec.vector.VectorizedRowBatch; 22 | 23 | /** Used for implementing ORC batch readers. */ 24 | public interface OrcBatchReader { 25 | 26 | /** Reads a row batch. */ 27 | T read(VectorizedRowBatch batch); 28 | 29 | void setBatchContext(long batchOffsetInFile); 30 | } 31 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/PartitionScanTask.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg; 20 | 21 | /** A scan task for data within a particular partition */ 22 | public interface PartitionScanTask extends ScanTask { 23 | /** Returns the spec of the partition for this scan task */ 24 | PartitionSpec spec(); 25 | 26 | /** Returns the value of the partition for this scan task */ 27 | StructLike partition(); 28 | } 29 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/ClientPool.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg; 20 | 21 | public interface ClientPool { 22 | interface Action { 23 | R run(C client) throws E; 24 | } 25 | 26 | R run(Action action) throws E, InterruptedException; 27 | 28 | R run(Action action, boolean retry) throws E, InterruptedException; 29 | } 30 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/StructLike.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg; 20 | 21 | /** 22 | * Interface for accessing data by position in a schema. 23 | * 24 | *

This interface supports accessing data in top-level fields, not in nested fields. 25 | */ 26 | public interface StructLike { 27 | int size(); 28 | 29 | T get(int pos, Class javaClass); 30 | 31 | void set(int pos, T value); 32 | } 33 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/UpdateLocation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg; 20 | 21 | /** API for setting a table's base location. */ 22 | public interface UpdateLocation extends PendingUpdate { 23 | /** 24 | * Set the table's location. 25 | * 26 | * @param location a String location 27 | * @return this for method chaining 28 | */ 29 | UpdateLocation setLocation(String location); 30 | } 31 | -------------------------------------------------------------------------------- /core/src/test/java/org/apache/iceberg/TestEnvironmentContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg; 20 | 21 | import org.assertj.core.api.Assertions; 22 | import org.junit.Test; 23 | 24 | public class TestEnvironmentContext { 25 | 26 | @Test 27 | public void testDefaultValue() { 28 | Assertions.assertThat(EnvironmentContext.get().get("iceberg-version")) 29 | .isEqualTo(IcebergBuild.fullVersion()); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/util/WriteDeltaProjections.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.spark.sql.catalyst.util 21 | 22 | import org.apache.spark.sql.catalyst.ProjectingInternalRow 23 | 24 | case class WriteDeltaProjections( 25 | rowProjection: Option[ProjectingInternalRow], 26 | rowIdProjection: ProjectingInternalRow, 27 | metadataProjection: Option[ProjectingInternalRow]) 28 | -------------------------------------------------------------------------------- /spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/util/WriteDeltaProjections.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.spark.sql.catalyst.util 21 | 22 | import org.apache.spark.sql.catalyst.ProjectingInternalRow 23 | 24 | case class WriteDeltaProjections( 25 | rowProjection: Option[ProjectingInternalRow], 26 | rowIdProjection: ProjectingInternalRow, 27 | metadataProjection: Option[ProjectingInternalRow]) 28 | -------------------------------------------------------------------------------- /spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestPartitionedWrites.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.spark.sql; 20 | 21 | import java.util.Map; 22 | 23 | public class TestPartitionedWrites extends PartitionedWritesTestBase { 24 | 25 | public TestPartitionedWrites( 26 | String catalogName, String implementation, Map config) { 27 | super(catalogName, implementation, config); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestUnpartitionedWrites.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.spark.sql; 20 | 21 | import java.util.Map; 22 | 23 | public class TestUnpartitionedWrites extends UnpartitionedWritesTestBase { 24 | 25 | public TestUnpartitionedWrites( 26 | String catalogName, String implementation, Map config) { 27 | super(catalogName, implementation, config); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /parquet/src/main/java/org/apache/iceberg/parquet/ParquetValueReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.parquet; 20 | 21 | import java.util.List; 22 | import org.apache.parquet.column.page.PageReadStore; 23 | 24 | public interface ParquetValueReader { 25 | T read(T reuse); 26 | 27 | TripleIterator column(); 28 | 29 | List> columns(); 30 | 31 | void setPageSource(PageReadStore pageStore, long rowPosition); 32 | } 33 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/FileContent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg; 20 | 21 | /** Content type stored in a file, one of DATA, POSITION_DELETES, or EQUALITY_DELETES. */ 22 | public enum FileContent { 23 | DATA(0), 24 | POSITION_DELETES(1), 25 | EQUALITY_DELETES(2); 26 | 27 | private final int id; 28 | 29 | FileContent(int id) { 30 | this.id = id; 31 | } 32 | 33 | public int id() { 34 | return id; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /flink/v1.14/flink-runtime/src/integration/java/org/apache/iceberg/flink/IcebergConnectorSmokeTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.flink; 20 | 21 | import java.util.Map; 22 | 23 | public class IcebergConnectorSmokeTest extends TestIcebergConnector { 24 | 25 | public IcebergConnectorSmokeTest( 26 | String catalogName, Map properties, boolean isStreaming) { 27 | super(catalogName, properties, isStreaming); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /flink/v1.14/flink/src/test/java/org/apache/iceberg/flink/util/TestFlinkPackage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.flink.util; 20 | 21 | import org.junit.Assert; 22 | import org.junit.Test; 23 | 24 | public class TestFlinkPackage { 25 | 26 | /** This unit test would need to be adjusted as new Flink version is supported. */ 27 | @Test 28 | public void testVersion() { 29 | Assert.assertEquals("1.14.0", FlinkPackage.version()); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /flink/v1.15/flink-runtime/src/integration/java/org/apache/iceberg/flink/IcebergConnectorSmokeTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.flink; 20 | 21 | import java.util.Map; 22 | 23 | public class IcebergConnectorSmokeTest extends TestIcebergConnector { 24 | 25 | public IcebergConnectorSmokeTest( 26 | String catalogName, Map properties, boolean isStreaming) { 27 | super(catalogName, properties, isStreaming); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /flink/v1.15/flink/src/test/java/org/apache/iceberg/flink/util/TestFlinkPackage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.flink.util; 20 | 21 | import org.junit.Assert; 22 | import org.junit.Test; 23 | 24 | public class TestFlinkPackage { 25 | 26 | /** This unit test would need to be adjusted as new Flink version is supported. */ 27 | @Test 28 | public void testVersion() { 29 | Assert.assertEquals("1.15.0", FlinkPackage.version()); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /flink/v1.16/flink-runtime/src/integration/java/org/apache/iceberg/flink/IcebergConnectorSmokeTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.flink; 20 | 21 | import java.util.Map; 22 | 23 | public class IcebergConnectorSmokeTest extends TestIcebergConnector { 24 | 25 | public IcebergConnectorSmokeTest( 26 | String catalogName, Map properties, boolean isStreaming) { 27 | super(catalogName, properties, isStreaming); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /flink/v1.16/flink/src/test/java/org/apache/iceberg/flink/util/TestFlinkPackage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.flink.util; 20 | 21 | import org.junit.Assert; 22 | import org.junit.Test; 23 | 24 | public class TestFlinkPackage { 25 | 26 | /** This unit test would need to be adjusted as new Flink version is supported. */ 27 | @Test 28 | public void testVersion() { 29 | Assert.assertEquals("1.16.0", FlinkPackage.version()); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /spark/v3.1/spark/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/UnspecifiedDistribution.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.spark.sql.connector.iceberg.distributions; 20 | 21 | import org.apache.spark.annotation.Experimental; 22 | 23 | /** 24 | * A distribution where no promises are made about co-location of data. 25 | * 26 | * @since 3.2.0 27 | */ 28 | @Experimental 29 | public interface UnspecifiedDistribution extends Distribution {} 30 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/puffin/StandardPuffinProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.puffin; 20 | 21 | public final class StandardPuffinProperties { 22 | private StandardPuffinProperties() {} 23 | 24 | /** 25 | * human-readable identification of the application writing the file, along with its version. 26 | * Example "Trino version 381". 27 | */ 28 | public static final String CREATED_BY_PROPERTY = "created-by"; 29 | } 30 | -------------------------------------------------------------------------------- /spark/v3.2/spark/src/main/java/org/apache/spark/sql/connector/iceberg/write/RowLevelOperationBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.spark.sql.connector.iceberg.write; 20 | 21 | /** An interface for building a row-level operation. */ 22 | public interface RowLevelOperationBuilder { 23 | /** 24 | * Returns a row-level operation that controls how Spark rewrites data for DELETE, UPDATE, MERGE 25 | * commands. 26 | */ 27 | RowLevelOperation build(); 28 | } 29 | -------------------------------------------------------------------------------- /core/src/test/resources/TableMetadataV2MissingSortOrder.json: -------------------------------------------------------------------------------- 1 | { 2 | "format-version": 2, 3 | "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", 4 | "location": "s3://bucket/test/location", 5 | "last-sequence-number": 34, 6 | "last-updated-ms": 1602638573590, 7 | "last-column-id": 3, 8 | "current-schema-id": 0, 9 | "schemas": [{ 10 | "type": "struct", 11 | "schema-id": 0, 12 | "fields": [ 13 | { 14 | "id": 1, 15 | "name": "x", 16 | "required": true, 17 | "type": "long" 18 | }, 19 | { 20 | "id": 2, 21 | "name": "y", 22 | "required": true, 23 | "type": "long", 24 | "doc": "comment" 25 | }, 26 | { 27 | "id": 3, 28 | "name": "z", 29 | "required": true, 30 | "type": "long" 31 | } 32 | ] 33 | }], 34 | "default-spec-id": 0, 35 | "partition-specs": [ 36 | { 37 | "spec-id": 0, 38 | "fields": [ 39 | { 40 | "name": "x", 41 | "transform": "identity", 42 | "source-id": 1, 43 | "field-id": 1000 44 | } 45 | ] 46 | } 47 | ], 48 | "last-partition-id": 1000, 49 | "properties": {}, 50 | "current-snapshot-id": -1, 51 | "snapshots": [], 52 | "snapshot-log": [], 53 | "metadata-log": [] 54 | } -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/puffin/StandardBlobTypes.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.puffin; 20 | 21 | public final class StandardBlobTypes { 22 | private StandardBlobTypes() {} 23 | 24 | /** 25 | * A serialized form of a "compact" Theta sketch produced by the Apache DataSketches library 27 | */ 28 | public static final String APACHE_DATASKETCHES_THETA_V1 = "apache-datasketches-theta-v1"; 29 | } 30 | -------------------------------------------------------------------------------- /spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RowLevelCommand.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.spark.sql.catalyst.plans.logical 21 | 22 | import org.apache.spark.sql.catalyst.expressions.Expression 23 | 24 | trait RowLevelCommand extends Command with SupportsSubquery { 25 | def condition: Option[Expression] 26 | def rewritePlan: Option[LogicalPlan] 27 | def withNewRewritePlan(newRewritePlan: LogicalPlan): RowLevelCommand 28 | } 29 | -------------------------------------------------------------------------------- /spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RowLevelCommand.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.spark.sql.catalyst.plans.logical 21 | 22 | import org.apache.spark.sql.catalyst.expressions.Expression 23 | 24 | trait RowLevelCommand extends Command with SupportsSubquery { 25 | def condition: Option[Expression] 26 | def rewritePlan: Option[LogicalPlan] 27 | def withNewRewritePlan(newRewritePlan: LogicalPlan): RowLevelCommand 28 | } 29 | -------------------------------------------------------------------------------- /flink/v1.14/flink/src/main/java/org/apache/iceberg/flink/source/assigner/SplitAssignerType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.flink.source.assigner; 20 | 21 | import org.apache.flink.annotation.Internal; 22 | 23 | @Internal 24 | public enum SplitAssignerType { 25 | SIMPLE { 26 | @Override 27 | public SplitAssignerFactory factory() { 28 | return new SimpleSplitAssignerFactory(); 29 | } 30 | }; 31 | 32 | public abstract SplitAssignerFactory factory(); 33 | } 34 | -------------------------------------------------------------------------------- /flink/v1.15/flink/src/main/java/org/apache/iceberg/flink/source/assigner/SplitAssignerType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.flink.source.assigner; 20 | 21 | import org.apache.flink.annotation.Internal; 22 | 23 | @Internal 24 | public enum SplitAssignerType { 25 | SIMPLE { 26 | @Override 27 | public SplitAssignerFactory factory() { 28 | return new SimpleSplitAssignerFactory(); 29 | } 30 | }; 31 | 32 | public abstract SplitAssignerFactory factory(); 33 | } 34 | -------------------------------------------------------------------------------- /flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/source/assigner/SplitAssignerType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.flink.source.assigner; 20 | 21 | import org.apache.flink.annotation.Internal; 22 | 23 | @Internal 24 | public enum SplitAssignerType { 25 | SIMPLE { 26 | @Override 27 | public SplitAssignerFactory factory() { 28 | return new SimpleSplitAssignerFactory(); 29 | } 30 | }; 31 | 32 | public abstract SplitAssignerFactory factory(); 33 | } 34 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/io/SupportsBulkOperations.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.io; 20 | 21 | public interface SupportsBulkOperations extends FileIO { 22 | /** 23 | * Delete the files at the given paths. 24 | * 25 | * @param pathsToDelete The paths to delete 26 | * @throws BulkDeletionFailureException in case of failure to delete at least 1 file 27 | */ 28 | void deleteFiles(Iterable pathsToDelete) throws BulkDeletionFailureException; 29 | } 30 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/deletes/DeleteCounter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.deletes; 20 | 21 | /** A counter to be used to count deletes as they are applied. */ 22 | public class DeleteCounter { 23 | 24 | private long count = 0L; 25 | 26 | /** Increment the counter by one. */ 27 | public void increment() { 28 | count++; 29 | } 30 | 31 | /** Return the current value of the counter. */ 32 | public long get() { 33 | return count; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/source/HasIcebergCatalog.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.spark.source; 20 | 21 | import org.apache.iceberg.catalog.Catalog; 22 | import org.apache.spark.sql.connector.catalog.TableCatalog; 23 | 24 | public interface HasIcebergCatalog extends TableCatalog { 25 | 26 | /** 27 | * Returns the underlying {@link org.apache.iceberg.catalog.Catalog} backing this Spark Catalog 28 | */ 29 | Catalog icebergCatalog(); 30 | } 31 | -------------------------------------------------------------------------------- /spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/source/HasIcebergCatalog.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.spark.source; 20 | 21 | import org.apache.iceberg.catalog.Catalog; 22 | import org.apache.spark.sql.connector.catalog.TableCatalog; 23 | 24 | public interface HasIcebergCatalog extends TableCatalog { 25 | 26 | /** 27 | * Returns the underlying {@link org.apache.iceberg.catalog.Catalog} backing this Spark Catalog 28 | */ 29 | Catalog icebergCatalog(); 30 | } 31 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/NullOrder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg; 20 | 21 | public enum NullOrder { 22 | NULLS_FIRST, 23 | NULLS_LAST; 24 | 25 | @Override 26 | public String toString() { 27 | switch (this) { 28 | case NULLS_FIRST: 29 | return "NULLS FIRST"; 30 | case NULLS_LAST: 31 | return "NULLS LAST"; 32 | default: 33 | throw new IllegalArgumentException("Unexpected null order: " + this); 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/util/SerializableFunction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.util; 20 | 21 | import java.io.Serializable; 22 | import java.util.function.Function; 23 | 24 | /** 25 | * A concrete transform function that applies a transform to values of a certain type. 26 | * 27 | * @param Java class of source values 28 | * @param Java class of transformed values 29 | */ 30 | public interface SerializableFunction extends Function, Serializable {} 31 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/expressions/Zorder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.expressions; 20 | 21 | import java.util.Arrays; 22 | import java.util.List; 23 | 24 | public class Zorder implements Term { 25 | private final NamedReference[] refs; 26 | 27 | public Zorder(List> refs) { 28 | this.refs = refs.toArray(new NamedReference[0]); 29 | } 30 | 31 | public List> refs() { 32 | return Arrays.asList(refs); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /core/src/test/java/org/apache/iceberg/rest/auth/AuthSessionUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.rest.auth; 20 | 21 | import org.apache.iceberg.rest.auth.OAuth2Util.AuthSession; 22 | 23 | /** Helper class to make the token refresh retries configurable for testing */ 24 | public class AuthSessionUtil { 25 | 26 | private AuthSessionUtil() {} 27 | 28 | public static void setTokenRefreshNumRetries(int retries) { 29 | AuthSession.setTokenRefreshNumRetries(retries); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /arrow/src/main/java/org/apache/iceberg/arrow/ArrowAllocation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.arrow; 20 | 21 | import org.apache.arrow.memory.RootAllocator; 22 | 23 | public class ArrowAllocation { 24 | static { 25 | ROOT_ALLOCATOR = new RootAllocator(Long.MAX_VALUE); 26 | } 27 | 28 | private static final RootAllocator ROOT_ALLOCATOR; 29 | 30 | private ArrowAllocation() {} 31 | 32 | public static RootAllocator rootAllocator() { 33 | return ROOT_ALLOCATOR; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/avro/ValueWriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.avro; 20 | 21 | import java.io.IOException; 22 | import java.util.stream.Stream; 23 | import org.apache.avro.io.Encoder; 24 | import org.apache.iceberg.FieldMetrics; 25 | 26 | public interface ValueWriter { 27 | void write(D datum, Encoder encoder) throws IOException; 28 | 29 | default Stream metrics() { 30 | return Stream.empty(); // TODO will populate in following PRs 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.aws.s3.signer; 20 | 21 | import java.net.URI; 22 | import java.util.List; 23 | import java.util.Map; 24 | import org.apache.iceberg.rest.RESTResponse; 25 | import org.immutables.value.Value; 26 | 27 | @Value.Immutable 28 | public interface S3SignResponse extends RESTResponse { 29 | URI uri(); 30 | 31 | Map> headers(); 32 | 33 | @Override 34 | default void validate() {} 35 | } 36 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/SplitScanTaskIterator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg; 20 | 21 | import java.util.Iterator; 22 | 23 | /** 24 | * An iterator that splits tasks. 25 | * 26 | * @param the Java type of tasks produced by this iterator 27 | */ 28 | interface SplitScanTaskIterator extends Iterator { 29 | 30 | @FunctionalInterface 31 | interface SplitScanTaskCreator { 32 | T create(T parentTask, long offset, long length); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /flink/v1.14/flink/src/main/java/org/apache/iceberg/flink/source/assigner/SplitAssignerFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.flink.source.assigner; 20 | 21 | import java.io.Serializable; 22 | import java.util.Collection; 23 | import org.apache.iceberg.flink.source.split.IcebergSourceSplitState; 24 | 25 | public interface SplitAssignerFactory extends Serializable { 26 | 27 | SplitAssigner createAssigner(); 28 | 29 | SplitAssigner createAssigner(Collection assignerState); 30 | } 31 | -------------------------------------------------------------------------------- /flink/v1.15/flink/src/main/java/org/apache/iceberg/flink/source/assigner/SplitAssignerFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.flink.source.assigner; 20 | 21 | import java.io.Serializable; 22 | import java.util.Collection; 23 | import org.apache.iceberg.flink.source.split.IcebergSourceSplitState; 24 | 25 | public interface SplitAssignerFactory extends Serializable { 26 | 27 | SplitAssigner createAssigner(); 28 | 29 | SplitAssigner createAssigner(Collection assignerState); 30 | } 31 | -------------------------------------------------------------------------------- /flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/source/assigner/SplitAssignerFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.flink.source.assigner; 20 | 21 | import java.io.Serializable; 22 | import java.util.Collection; 23 | import org.apache.iceberg.flink.source.split.IcebergSourceSplitState; 24 | 25 | public interface SplitAssignerFactory extends Serializable { 26 | 27 | SplitAssigner createAssigner(); 28 | 29 | SplitAssigner createAssigner(Collection assignerState); 30 | } 31 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | jmhOutputPath=build/reports/jmh/human-readable-output.txt 17 | jmhJsonOutputPath=build/reports/jmh/results.json 18 | jmhIncludeRegex=.* 19 | systemProp.defaultFlinkVersions=1.16 20 | systemProp.knownFlinkVersions=1.14,1.15,1.16 21 | systemProp.defaultHiveVersions= 22 | systemProp.knownHiveVersions=3 23 | systemProp.defaultSparkVersions=3.3 24 | systemProp.knownSparkVersions=2.4,3.1,3.2,3.3 25 | systemProp.defaultScalaVersion=2.12 26 | systemProp.knownScalaVersions=2.12,2.13 27 | org.gradle.parallel=true 28 | org.gradle.jvmargs=-Xmx1024m 29 | -------------------------------------------------------------------------------- /mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/WriteObjectInspector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.mr.hive.serde.objectinspector; 20 | 21 | /** 22 | * Interface for converting the Hive primitive objects for the objects which could be added to an 23 | * Iceberg Record. If the IcebergObjectInspector does not implement this then the default Hive 24 | * primitive objects will be used without conversion. 25 | */ 26 | public interface WriteObjectInspector { 27 | Object convert(Object value); 28 | } 29 | -------------------------------------------------------------------------------- /flink/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | def flinkVersions = (System.getProperty("flinkVersions") != null ? System.getProperty("flinkVersions") : System.getProperty("defaultFlinkVersions")).split(",") 21 | 22 | if (flinkVersions.contains("1.14")) { 23 | apply from: file("$projectDir/v1.14/build.gradle") 24 | } 25 | 26 | if (flinkVersions.contains("1.15")) { 27 | apply from: file("$projectDir/v1.15/build.gradle") 28 | } 29 | 30 | if (flinkVersions.contains("1.16")) { 31 | apply from: file("$projectDir/v1.16/build.gradle") 32 | } 33 | -------------------------------------------------------------------------------- /hive-metastore/src/main/java/org/apache/iceberg/hive/LockException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.hive; 20 | 21 | import com.google.errorprone.annotations.FormatMethod; 22 | 23 | class LockException extends RuntimeException { 24 | @FormatMethod 25 | LockException(String message, Object... args) { 26 | super(String.format(message, args)); 27 | } 28 | 29 | @FormatMethod 30 | LockException(Throwable cause, String message, Object... args) { 31 | super(String.format(message, args), cause); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | version: 2 21 | updates: 22 | - package-ecosystem: "pip" 23 | directory: "/python/" 24 | schedule: 25 | interval: "weekly" 26 | day: "sunday" 27 | open-pull-requests-limit: 5 28 | - package-ecosystem: "github-actions" 29 | directory: "/" 30 | schedule: 31 | interval: "weekly" 32 | day: "sunday" 33 | - package-ecosystem: "gradle" 34 | directory: "/" 35 | schedule: 36 | interval: "weekly" 37 | day: "sunday" 38 | open-pull-requests-limit: 5 39 | 40 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/util/NaNUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.util; 20 | 21 | public class NaNUtil { 22 | 23 | private NaNUtil() {} 24 | 25 | public static boolean isNaN(Object value) { 26 | if (value == null) { 27 | return false; 28 | } 29 | 30 | if (value instanceof Double) { 31 | return Double.isNaN((Double) value); 32 | } else if (value instanceof Float) { 33 | return Float.isNaN((Float) value); 34 | } else { 35 | return false; 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/actions/BaseMigrateTableActionResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.actions; 20 | 21 | public class BaseMigrateTableActionResult implements MigrateTable.Result { 22 | 23 | private final long migratedDataFilesCount; 24 | 25 | public BaseMigrateTableActionResult(long migratedDataFilesCount) { 26 | this.migratedDataFilesCount = migratedDataFilesCount; 27 | } 28 | 29 | @Override 30 | public long migratedDataFilesCount() { 31 | return migratedDataFilesCount; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /spark/v3.2/spark/src/main/java/org/apache/spark/sql/connector/iceberg/write/DeltaWriteBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.spark.sql.connector.iceberg.write; 20 | 21 | import org.apache.spark.sql.connector.write.WriteBuilder; 22 | 23 | /** An interface for building delta writes. */ 24 | public interface DeltaWriteBuilder extends WriteBuilder { 25 | /** Returns a logical delta write. */ 26 | @Override 27 | default DeltaWrite build() { 28 | throw new UnsupportedOperationException("Not implemented: build"); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /spark/v3.3/spark/src/main/java/org/apache/spark/sql/connector/iceberg/write/DeltaWriteBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.spark.sql.connector.iceberg.write; 20 | 21 | import org.apache.spark.sql.connector.write.WriteBuilder; 22 | 23 | /** An interface for building delta writes. */ 24 | public interface DeltaWriteBuilder extends WriteBuilder { 25 | /** Returns a logical delta write. */ 26 | @Override 27 | default DeltaWrite build() { 28 | throw new UnsupportedOperationException("Not implemented: build"); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/actions/BaseSnapshotTableActionResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.actions; 20 | 21 | public class BaseSnapshotTableActionResult implements SnapshotTable.Result { 22 | 23 | private final long importedDataFilesCount; 24 | 25 | public BaseSnapshotTableActionResult(long importedDataFilesCount) { 26 | this.importedDataFilesCount = importedDataFilesCount; 27 | } 28 | 29 | @Override 30 | public long importedDataFilesCount() { 31 | return importedDataFilesCount; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/rest/RESTMessage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.rest; 20 | 21 | /** Interface to mark both REST requests and responses. */ 22 | public interface RESTMessage { 23 | 24 | /** 25 | * Ensures that a constructed instance of a REST message is valid according to the REST spec. 26 | * 27 | *

This is needed when parsing data that comes from external sources and the object might have 28 | * been constructed without all the required fields present. 29 | */ 30 | void validate(); 31 | } 32 | -------------------------------------------------------------------------------- /flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/FlinkEnvironmentContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.flink; 20 | 21 | import org.apache.iceberg.EnvironmentContext; 22 | import org.apache.iceberg.flink.util.FlinkPackage; 23 | 24 | class FlinkEnvironmentContext { 25 | private FlinkEnvironmentContext() {} 26 | 27 | public static void init() { 28 | EnvironmentContext.put(EnvironmentContext.ENGINE_NAME, "flink"); 29 | EnvironmentContext.put(EnvironmentContext.ENGINE_VERSION, FlinkPackage.version()); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/connector/write/RowLevelOperationInfoImpl.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.spark.sql.connector.write 21 | 22 | import org.apache.spark.sql.connector.iceberg.write.RowLevelOperation.Command 23 | import org.apache.spark.sql.connector.iceberg.write.RowLevelOperationInfo 24 | import org.apache.spark.sql.util.CaseInsensitiveStringMap 25 | 26 | case class RowLevelOperationInfoImpl( 27 | command: Command, 28 | options: CaseInsensitiveStringMap) extends RowLevelOperationInfo 29 | -------------------------------------------------------------------------------- /spark/v3.2/spark/src/main/java/org/apache/spark/sql/connector/iceberg/write/DeltaBatchWrite.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.spark.sql.connector.iceberg.write; 20 | 21 | import org.apache.spark.sql.connector.write.BatchWrite; 22 | import org.apache.spark.sql.connector.write.PhysicalWriteInfo; 23 | 24 | /** An interface that defines how to write a delta of rows during batch processing. */ 25 | public interface DeltaBatchWrite extends BatchWrite { 26 | @Override 27 | DeltaWriterFactory createBatchWriterFactory(PhysicalWriteInfo info); 28 | } 29 | -------------------------------------------------------------------------------- /spark/v3.3/spark/src/main/java/org/apache/spark/sql/connector/iceberg/write/DeltaBatchWrite.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.spark.sql.connector.iceberg.write; 20 | 21 | import org.apache.spark.sql.connector.write.BatchWrite; 22 | import org.apache.spark.sql.connector.write.PhysicalWriteInfo; 23 | 24 | /** An interface that defines how to write a delta of rows during batch processing. */ 25 | public interface DeltaBatchWrite extends BatchWrite { 26 | @Override 27 | DeltaWriterFactory createBatchWriterFactory(PhysicalWriteInfo info); 28 | } 29 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/io/PositionOutputStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.io; 20 | 21 | import java.io.IOException; 22 | import java.io.OutputStream; 23 | 24 | public abstract class PositionOutputStream extends OutputStream { 25 | /** 26 | * Return the current position in the OutputStream. 27 | * 28 | * @return current position in bytes from the start of the stream 29 | * @throws IOException If the underlying stream throws IOException 30 | */ 31 | public abstract long getPos() throws IOException; 32 | } 33 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/avro/MetricsAwareDatumWriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.avro; 20 | 21 | import java.util.stream.Stream; 22 | import org.apache.avro.io.DatumWriter; 23 | import org.apache.iceberg.FieldMetrics; 24 | 25 | /** Wrapper writer around {@link DatumWriter} with metrics support. */ 26 | public interface MetricsAwareDatumWriter extends DatumWriter { 27 | 28 | /** Returns a stream of {@link FieldMetrics} that this MetricsAwareDatumWriter keeps track of. */ 29 | Stream metrics(); 30 | } 31 | -------------------------------------------------------------------------------- /flink/v1.14/flink/src/main/java/org/apache/iceberg/flink/FlinkEnvironmentContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.flink; 20 | 21 | import org.apache.iceberg.EnvironmentContext; 22 | import org.apache.iceberg.flink.util.FlinkPackage; 23 | 24 | class FlinkEnvironmentContext { 25 | 26 | private FlinkEnvironmentContext() {} 27 | 28 | public static void init() { 29 | EnvironmentContext.put(EnvironmentContext.ENGINE_NAME, "flink"); 30 | EnvironmentContext.put(EnvironmentContext.ENGINE_VERSION, FlinkPackage.version()); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /flink/v1.15/flink/src/main/java/org/apache/iceberg/flink/FlinkEnvironmentContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.flink; 20 | 21 | import org.apache.iceberg.EnvironmentContext; 22 | import org.apache.iceberg.flink.util.FlinkPackage; 23 | 24 | class FlinkEnvironmentContext { 25 | 26 | private FlinkEnvironmentContext() {} 27 | 28 | public static void init() { 29 | EnvironmentContext.put(EnvironmentContext.ENGINE_NAME, "flink"); 30 | EnvironmentContext.put(EnvironmentContext.ENGINE_VERSION, FlinkPackage.version()); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /spark/v3.2/spark/src/main/java/org/apache/spark/sql/connector/iceberg/write/DeltaWriterFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.spark.sql.connector.iceberg.write; 20 | 21 | import org.apache.spark.sql.catalyst.InternalRow; 22 | import org.apache.spark.sql.connector.write.DataWriterFactory; 23 | 24 | /** A factory for creating and initializing delta writers at the executor side. */ 25 | public interface DeltaWriterFactory extends DataWriterFactory { 26 | @Override 27 | DeltaWriter createWriter(int partitionId, long taskId); 28 | } 29 | -------------------------------------------------------------------------------- /spark/v3.3/spark/src/main/java/org/apache/spark/sql/connector/iceberg/write/DeltaWriterFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.spark.sql.connector.iceberg.write; 20 | 21 | import org.apache.spark.sql.catalyst.InternalRow; 22 | import org.apache.spark.sql.connector.write.DataWriterFactory; 23 | 24 | /** A factory for creating and initializing delta writers at the executor side. */ 25 | public interface DeltaWriterFactory extends DataWriterFactory { 26 | @Override 27 | DeltaWriter createWriter(int partitionId, long taskId); 28 | } 29 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/io/BulkDeletionFailureException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.io; 20 | 21 | public class BulkDeletionFailureException extends RuntimeException { 22 | private final int numberFailedObjects; 23 | 24 | public BulkDeletionFailureException(int numberFailedObjects) { 25 | super(String.format("Failed to delete %d files", numberFailedObjects)); 26 | this.numberFailedObjects = numberFailedObjects; 27 | } 28 | 29 | public int numberFailedObjects() { 30 | return numberFailedObjects; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /api/src/test/java/org/apache/iceberg/TestTransformSerialization.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg; 20 | 21 | import org.junit.Assert; 22 | import org.junit.Test; 23 | 24 | public class TestTransformSerialization extends PartitionSpecTestBase { 25 | @Test 26 | public void testTransforms() throws Exception { 27 | for (PartitionSpec spec : SPECS) { 28 | Assert.assertEquals( 29 | "Deserialization should produce equal partition spec", 30 | spec, 31 | TestHelpers.roundTripSerialize(spec)); 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/io/FileIOMetricsContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.io; 20 | 21 | import org.apache.iceberg.metrics.MetricsContext; 22 | 23 | /** 24 | * Extension of MetricsContext for use with FileIO to define standard metrics that should be 25 | * reported. 26 | */ 27 | public interface FileIOMetricsContext extends MetricsContext { 28 | String READ_BYTES = "read.bytes"; 29 | String READ_OPERATIONS = "read.operations"; 30 | String WRITE_BYTES = "write.bytes"; 31 | String WRITE_OPERATIONS = "write.operations"; 32 | } 33 | -------------------------------------------------------------------------------- /delta-lake/src/main/java/org/apache/iceberg/delta/BaseSnapshotDeltaLakeTableActionResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.delta; 20 | 21 | class BaseSnapshotDeltaLakeTableActionResult implements SnapshotDeltaLakeTable.Result { 22 | 23 | private final long snapshotDataFilesCount; 24 | 25 | BaseSnapshotDeltaLakeTableActionResult(long snapshotDataFilesCount) { 26 | this.snapshotDataFilesCount = snapshotDataFilesCount; 27 | } 28 | 29 | @Override 30 | public long snapshotDataFilesCount() { 31 | return snapshotDataFilesCount; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/actions/BaseDeleteOrphanFilesActionResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.actions; 20 | 21 | public class BaseDeleteOrphanFilesActionResult implements DeleteOrphanFiles.Result { 22 | 23 | private final Iterable orphanFileLocations; 24 | 25 | public BaseDeleteOrphanFilesActionResult(Iterable orphanFileLocations) { 26 | this.orphanFileLocations = orphanFileLocations; 27 | } 28 | 29 | @Override 30 | public Iterable orphanFileLocations() { 31 | return orphanFileLocations; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/encryption/EncryptionKeyMetadatas.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.encryption; 20 | 21 | import java.nio.ByteBuffer; 22 | 23 | public class EncryptionKeyMetadatas { 24 | 25 | public static EncryptionKeyMetadata of(ByteBuffer keyMetadata) { 26 | return BaseEncryptionKeyMetadata.fromKeyMetadata(keyMetadata); 27 | } 28 | 29 | public static EncryptionKeyMetadata of(byte[] keyMetadata) { 30 | return BaseEncryptionKeyMetadata.fromByteArray(keyMetadata); 31 | } 32 | 33 | private EncryptionKeyMetadatas() {} 34 | } 35 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/HistoryEntry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg; 20 | 21 | import java.io.Serializable; 22 | 23 | /** 24 | * Table history entry. 25 | * 26 | *

An entry contains a change to the table state. At the given timestamp, the current snapshot 27 | * was set to the given snapshot ID. 28 | */ 29 | public interface HistoryEntry extends Serializable { 30 | /** Returns the timestamp in milliseconds of the change. */ 31 | long timestampMillis(); 32 | 33 | /** Returns ID of the new current snapshot. */ 34 | long snapshotId(); 35 | } 36 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/exceptions/DuplicateWAPCommitException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.exceptions; 20 | 21 | /** 22 | * This exception occurs when the WAP workflow detects a duplicate wap commit. This helps clients to 23 | * detect duplicate snapshots that are connected by the same wap id. 24 | */ 25 | public class DuplicateWAPCommitException extends ValidationException { 26 | 27 | public DuplicateWAPCommitException(String wapId) { 28 | super("Duplicate request to cherry pick wap id that was published already: %s", wapId); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/jdbc/UncheckedSQLException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.jdbc; 20 | 21 | import com.google.errorprone.annotations.FormatMethod; 22 | 23 | public class UncheckedSQLException extends RuntimeException { 24 | 25 | @FormatMethod 26 | public UncheckedSQLException(String message, Object... args) { 27 | super(String.format(message, args)); 28 | } 29 | 30 | @FormatMethod 31 | public UncheckedSQLException(Throwable cause, String message, Object... args) { 32 | super(String.format(message, args), cause); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/metrics/CommitReport.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.metrics; 20 | 21 | import java.util.Map; 22 | import org.immutables.value.Value; 23 | 24 | /** A commit report that contains all relevant information from a Snapshot. */ 25 | @Value.Immutable 26 | public interface CommitReport extends MetricsReport { 27 | 28 | String tableName(); 29 | 30 | long snapshotId(); 31 | 32 | long sequenceNumber(); 33 | 34 | String operation(); 35 | 36 | CommitMetricsResult commitMetrics(); 37 | 38 | Map metadata(); 39 | } 40 | -------------------------------------------------------------------------------- /aliyun/src/test/java/org/apache/iceberg/aliyun/oss/mock/Range.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.aliyun.oss.mock; 20 | 21 | public class Range { 22 | 23 | private final long start; 24 | private final long end; 25 | 26 | public Range(long start, long end) { 27 | this.start = start; 28 | this.end = end; 29 | } 30 | 31 | public long start() { 32 | return start; 33 | } 34 | 35 | public long end() { 36 | return end; 37 | } 38 | 39 | @Override 40 | public String toString() { 41 | return String.format("%d-%d", start, end); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskNumSplits.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.spark.source.metrics; 20 | 21 | import org.apache.spark.sql.connector.metric.CustomTaskMetric; 22 | 23 | public class TaskNumSplits implements CustomTaskMetric { 24 | private final long value; 25 | 26 | public TaskNumSplits(long value) { 27 | this.value = value; 28 | } 29 | 30 | @Override 31 | public String name() { 32 | return "numSplits"; 33 | } 34 | 35 | @Override 36 | public long value() { 37 | return value; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskNumSplits.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.spark.source.metrics; 20 | 21 | import org.apache.spark.sql.connector.metric.CustomTaskMetric; 22 | 23 | public class TaskNumSplits implements CustomTaskMetric { 24 | private final long value; 25 | 26 | public TaskNumSplits(long value) { 27 | this.value = value; 28 | } 29 | 30 | @Override 31 | public String name() { 32 | return "numSplits"; 33 | } 34 | 35 | @Override 36 | public long value() { 37 | return value; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /spark/v3.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/NoStatsUnaryNode.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.spark.sql.catalyst.plans.logical 21 | 22 | import org.apache.spark.sql.catalyst.expressions.Attribute 23 | 24 | case class NoStatsUnaryNode(child: LogicalPlan) extends UnaryNode { 25 | override def output: Seq[Attribute] = child.output 26 | override def stats: Statistics = Statistics(Long.MaxValue) 27 | 28 | override protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = { 29 | copy(child = newChild) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskNumDeletes.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.spark.source.metrics; 20 | 21 | import org.apache.spark.sql.connector.metric.CustomTaskMetric; 22 | 23 | public class TaskNumDeletes implements CustomTaskMetric { 24 | private final long value; 25 | 26 | public TaskNumDeletes(long value) { 27 | this.value = value; 28 | } 29 | 30 | @Override 31 | public String name() { 32 | return "numDeletes"; 33 | } 34 | 35 | @Override 36 | public long value() { 37 | return value; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/NoStatsUnaryNode.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.spark.sql.catalyst.plans.logical 21 | 22 | import org.apache.spark.sql.catalyst.expressions.Attribute 23 | 24 | case class NoStatsUnaryNode(child: LogicalPlan) extends UnaryNode { 25 | override def output: Seq[Attribute] = child.output 26 | override def stats: Statistics = Statistics(Long.MaxValue) 27 | 28 | override protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = { 29 | copy(child = newChild) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/source/metrics/TaskNumDeletes.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.spark.source.metrics; 20 | 21 | import org.apache.spark.sql.connector.metric.CustomTaskMetric; 22 | 23 | public class TaskNumDeletes implements CustomTaskMetric { 24 | private final long value; 25 | 26 | public TaskNumDeletes(long value) { 27 | this.value = value; 28 | } 29 | 30 | @Override 31 | public String name() { 32 | return "numDeletes"; 33 | } 34 | 35 | @Override 36 | public long value() { 37 | return value; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/DataTask.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg; 20 | 21 | import org.apache.iceberg.io.CloseableIterable; 22 | 23 | /** A task that returns data as {@link StructLike rows} instead of where to read data. */ 24 | public interface DataTask extends FileScanTask { 25 | @Override 26 | default boolean isDataTask() { 27 | return true; 28 | } 29 | 30 | @Override 31 | default DataTask asDataTask() { 32 | return this; 33 | } 34 | 35 | /** Returns an iterable of {@link StructLike} rows. */ 36 | CloseableIterable rows(); 37 | } 38 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/view/ViewHistoryEntry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.view; 20 | 21 | import org.immutables.value.Value; 22 | 23 | /** 24 | * View history entry. 25 | * 26 | *

An entry contains a change to the view state. At the given timestamp, the current version was 27 | * set to the given version ID. 28 | */ 29 | @Value.Immutable 30 | public interface ViewHistoryEntry { 31 | /** Return the timestamp in milliseconds of the change */ 32 | long timestampMillis(); 33 | 34 | /** Return ID of the new current version */ 35 | int versionId(); 36 | } 37 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/exceptions/RESTException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.exceptions; 20 | 21 | import com.google.errorprone.annotations.FormatMethod; 22 | 23 | /** Base class for REST client exceptions */ 24 | public class RESTException extends RuntimeException { 25 | @FormatMethod 26 | public RESTException(String message, Object... args) { 27 | super(String.format(message, args)); 28 | } 29 | 30 | @FormatMethod 31 | public RESTException(Throwable cause, String message, Object... args) { 32 | super(String.format(message, args), cause); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /flink/v1.14/flink/src/main/java/org/apache/iceberg/flink/util/FlinkPackage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.flink.util; 20 | 21 | import org.apache.flink.streaming.api.datastream.DataStream; 22 | 23 | public class FlinkPackage { 24 | /** Choose {@link DataStream} class because it is one of the core Flink API. */ 25 | private static final String VERSION = DataStream.class.getPackage().getImplementationVersion(); 26 | 27 | private FlinkPackage() {} 28 | 29 | /** Returns Flink version string like x.y.z */ 30 | public static String version() { 31 | return VERSION; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /flink/v1.15/flink/src/main/java/org/apache/iceberg/flink/util/FlinkPackage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.flink.util; 20 | 21 | import org.apache.flink.streaming.api.datastream.DataStream; 22 | 23 | public class FlinkPackage { 24 | /** Choose {@link DataStream} class because it is one of the core Flink API. */ 25 | private static final String VERSION = DataStream.class.getPackage().getImplementationVersion(); 26 | 27 | private FlinkPackage() {} 28 | 29 | /** Returns Flink version string like x.y.z */ 30 | public static String version() { 31 | return VERSION; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /flink/v1.16/flink/src/main/java/org/apache/iceberg/flink/util/FlinkPackage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.flink.util; 20 | 21 | import org.apache.flink.streaming.api.datastream.DataStream; 22 | 23 | public class FlinkPackage { 24 | /** Choose {@link DataStream} class because it is one of the core Flink API. */ 25 | private static final String VERSION = DataStream.class.getPackage().getImplementationVersion(); 26 | 27 | private FlinkPackage() {} 28 | 29 | /** Returns Flink version string like x.y.z */ 30 | public static String version() { 31 | return VERSION; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/jdbc/UncheckedInterruptedException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.jdbc; 20 | 21 | import com.google.errorprone.annotations.FormatMethod; 22 | 23 | public class UncheckedInterruptedException extends RuntimeException { 24 | 25 | @FormatMethod 26 | public UncheckedInterruptedException(String message, Object... args) { 27 | super(String.format(message, args)); 28 | } 29 | 30 | @FormatMethod 31 | public UncheckedInterruptedException(Throwable cause, String message, Object... args) { 32 | super(String.format(message, args), cause); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/exceptions/ServiceFailureException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.exceptions; 20 | 21 | import com.google.errorprone.annotations.FormatMethod; 22 | 23 | /** Exception thrown on HTTP 5XX Server Error. */ 24 | public class ServiceFailureException extends RESTException { 25 | @FormatMethod 26 | public ServiceFailureException(String message, Object... args) { 27 | super(message, args); 28 | } 29 | 30 | @FormatMethod 31 | public ServiceFailureException(Throwable cause, String message, Object... args) { 32 | super(cause, message, args); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.aws.s3.signer; 20 | 21 | import java.net.URI; 22 | import java.util.List; 23 | import java.util.Map; 24 | import org.apache.iceberg.rest.RESTRequest; 25 | import org.immutables.value.Value; 26 | 27 | @Value.Immutable 28 | public interface S3SignRequest extends RESTRequest { 29 | String region(); 30 | 31 | String method(); 32 | 33 | URI uri(); 34 | 35 | Map> headers(); 36 | 37 | Map properties(); 38 | 39 | @Override 40 | default void validate() {} 41 | } 42 | -------------------------------------------------------------------------------- /spark/v3.3/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropTag.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.spark.sql.catalyst.plans.logical 21 | 22 | import org.apache.spark.sql.catalyst.expressions.Attribute 23 | 24 | case class DropTag(table: Seq[String], tag: String, ifExists: Boolean) extends LeafCommand { 25 | 26 | import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ 27 | 28 | override lazy val output: Seq[Attribute] = Nil 29 | 30 | override def simpleString(maxFields: Int): String = { 31 | s"DropTag tag: ${tag} for table: ${table.quoted}" 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /orc/src/main/java/org/apache/iceberg/orc/OrcValueReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.orc; 20 | 21 | import org.apache.orc.storage.ql.exec.vector.ColumnVector; 22 | 23 | public interface OrcValueReader { 24 | default T read(ColumnVector vector, int row) { 25 | int rowIndex = vector.isRepeating ? 0 : row; 26 | if (!vector.noNulls && vector.isNull[rowIndex]) { 27 | return null; 28 | } else { 29 | return nonNullRead(vector, rowIndex); 30 | } 31 | } 32 | 33 | T nonNullRead(ColumnVector vector, int row); 34 | 35 | default void setBatchContext(long batchOffsetInFile) {} 36 | } 37 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/iceberg/exceptions/BadRequestException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.exceptions; 20 | 21 | import com.google.errorprone.annotations.FormatMethod; 22 | 23 | /** Exception thrown on HTTP 400 - Bad Request */ 24 | public class BadRequestException extends RuntimeException { 25 | @FormatMethod 26 | public BadRequestException(String message, Object... args) { 27 | super(String.format(message, args)); 28 | } 29 | 30 | @FormatMethod 31 | public BadRequestException(Throwable cause, String message, Object... args) { 32 | super(String.format(message, args), cause); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /core/src/main/java/org/apache/iceberg/util/LocationUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.iceberg.util; 20 | 21 | import org.apache.iceberg.relocated.com.google.common.base.Preconditions; 22 | 23 | public class LocationUtil { 24 | private LocationUtil() {} 25 | 26 | public static String stripTrailingSlash(String path) { 27 | Preconditions.checkArgument( 28 | path != null && path.length() > 0, "path must not be null or empty"); 29 | 30 | String result = path; 31 | while (result.endsWith("/")) { 32 | result = result.substring(0, result.length() - 1); 33 | } 34 | return result; 35 | } 36 | } 37 | --------------------------------------------------------------------------------